Index: branches/simtest_nebulous_branches/ippTasks/dist.pro
===================================================================
--- branches/simtest_nebulous_branches/ippTasks/dist.pro	(revision 25061)
+++ branches/simtest_nebulous_branches/ippTasks/dist.pro	(revision 27840)
@@ -15,4 +15,19 @@
 $distToAdvance_DB = 0
 $distQueue_DB = 0
+$dist_revert_DB = 0
+
+### list of stages
+#list of stages
+$DIST_STAGE:n = 0
+list DIST_STAGE -add "raw"
+list DIST_STAGE -add "chip"
+list DIST_STAGE -add "camera"
+list DIST_STAGE -add "fake"
+list DIST_STAGE -add "warp"
+list DIST_STAGE -add "diff"
+list DIST_STAGE -add "stack"
+list DIST_STAGE -add "SSdiff"
+
+$currentStage = 0
 
 ### Check status of tasks
@@ -42,10 +57,9 @@
     active true
   end
-  task dist.queueruns
-#   We aren't ready to run this task yet. It will queue much too much 
-#    active true
-    active false
-  end
-end
+  task dist.revert
+    active false
+  end
+end
+
 macro dist.off
   task dist.process.load
@@ -61,8 +75,93 @@
     active false
   end
-  task dist.queueruns
-    active false
-  end
-end
+  task dist.revert
+    active false
+  end
+end
+
+macro dist.revert.on
+  task dist.revert
+    active true
+  end
+end
+
+macro dist.revert.off
+  task dist.revert
+    active false
+  end
+end
+
+macro get.host.for.component
+    if ($0 != 3)
+        echo "USAGE: get.host.for.component (componentID) (varname)"
+        break
+    end
+    local component host varname
+    $component = $1
+    $varname = $2
+    substr $component 0 3 COMP_HEAD
+    if ("$COMP_HEAD" == "sky")
+        book getword ipphosts dist_skycell count -var count
+        local j myValue skyhash
+        list word -splitbychar . $component
+        $skyhash = 0
+        for j 0 $word:n
+            inthash $word:$j $count -var myValue
+            $skyhash = $skyhash + $myValue
+        end
+        inthash $skyhash $count -var skyhash
+        sprintf skyname "sky%02d" $skyhash
+        book getword ipphosts dist_skycell $skyname -var host
+    else
+        book getword ipphosts dist_chip $component -var host
+    end
+    $$varname = $host
+end
+
+macro set.dist.workdir.by.component
+    if ($0 != 5)
+        echo "USAGE: set.workdir.by.component (stage_id) (componentID) (template) (varname)"
+        break
+    end
+    local host stage_id component default template varname length start count selector component_id random_number
+
+    $stage_id = $1
+    $component = $2
+    $template = $3
+    $varname = $4
+    if ("$template" == "NULL")
+        echo "ERROR: WORKDIR template not set."
+        break
+    end
+    book getword ipphosts distribution count -var count
+    if ("$count" == "NULL")
+        echo "ERROR: distributin hosts list is empty"
+        break
+    end
+    $host = "foo"
+    if ("$component" == "exposure") 
+        # take last two letters of selector
+        # treat it as an integer and use modulous of length of distribution hosts
+        # to compute an index into the host table
+        strlen $stage_id length
+        $start = $length - 2
+        substr $stage_id $start 2 index
+        $component_id = $index % $count
+        book getword ipphosts distribution $component_id -var myhost
+    else
+        get.host.for.component $component myhost
+    end
+
+    if ("$myhost" == "NULL")
+        echo "ERROR: failed to find host for $component"
+        break
+    end
+
+    strsub $template @HOST@ $myhost -var $varname
+
+    #echo template is $template
+    #echo outdir is $$varname
+end
+
 
 task	       dist.process.load
@@ -74,9 +173,13 @@
   npending     1
 
-  stdout NULL
-  stderr $LOGDIR/dist.process.log
 
   task.exec
-    $run = disttool -pendingcomponent
+     # stdout NULL
+     # stderr $LOGSUBDIR/dist.process.load.log
+
+    $run = disttool -pendingcomponent -stage $DIST_STAGE:$currentStage
+    $currentStage ++
+    if ($currentStage >= $DIST_STAGE:n) set currentStage = 0
+
     if ($DB:n == 0)
       option DEFAULT
@@ -85,6 +188,11 @@
       option $DB:$distToProcess_DB
       $run = $run -dbname $DB:$distToProcess_DB
-      $distToProcess_DB ++
-      if ($distToProcess_DB >= $DB:n) set distToProcess_DB = 0
+
+      # only increment the database number after we have gone through all of
+      # the stages
+      if ($currentStage == 0)
+          $distToProcess_DB ++
+          if ($distToProcess_DB >= $DB:n) set distToProcess_DB = 0
+      end
     end
     add_poll_args run
@@ -122,4 +230,5 @@
 
   task.exec
+    if ($NETWORK == 0) break
     book npages distToProcess -var N
     if ($N == 0) 
@@ -128,6 +237,4 @@
     end
     periods -exec 0.05
-
-    if ($NETWORK == 0) break
     
     # look for new components to process (pantaskState == INIT)
@@ -135,9 +242,11 @@
     if ("$pageName" == "NULL") break
 
+#    echo running $pageName
+
     book setword distToProcess $pageName pantaskState RUN
     book getword distToProcess $pageName dist_id -var DIST_ID
     book getword distToProcess $pageName camera -var CAMERA
-    book getword distToProcess $pageName stage -var STAGE
-    book getword distToProcess $pageName stage_id -var STAGE_ID
+    book getword distToProcess $pageName stage -var DIST_STAGE
+    book getword distToProcess $pageName stage_id -var DIST_STAGE_ID
     book getword distToProcess $pageName clean -var CLEAN
     book getword distToProcess $pageName component -var COMPONENT
@@ -149,5 +258,6 @@
     book getword distToProcess $pageName no_magic -var NO_MAGIC
     book getword distToProcess $pageName magicked -var MAGICKED
-    book getword distToProcess $pageName outdir -var OUTDIR
+    book getword distToProcess $pageName alt_path_base -var ALT_PATH_BASE
+    book getword distToProcess $pageName outdir -var OUTDIR_TEMPLATE
     book getword distToProcess $pageName dbname -var DBNAME
 
@@ -160,6 +270,6 @@
         $EXTRA_ARGS = $EXTRA_ARGS --magicked
     end
-    # no_magic is output as integer due to the union in the sql
-    if ($NO_MAGIC)
+    # is this right for stack and fake?
+    if ("$NO_MAGIC" == "T")
         $EXTRA_ARGS = $EXTRA_ARGS --no_magic
     end
@@ -167,9 +277,14 @@
         $EXTRA_ARGS = $EXTRA_ARGS --poor_quality
     end
-
-#    set.host.for.camera $CAMERA $MAGIC_ID
-#    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
-#    host anyhost
-
+    if ("$ALT_PATH_BASE" != "")
+        $EXTRA_ARGS = $EXTRA_ARGS --alt_path_base $ALT_PATH_BASE
+    end
+
+
+    set.dist.workdir.by.component $DIST_STAGE_ID $COMPONENT $OUTDIR_TEMPLATE OUTDIR 
+    if ("$OUTDIR" == "NULL")
+        echo ERROR failed to set workdir for $COMPONENT
+        break
+    end
 
     substr $COMPONENT 0 3 COMP_HEAD
@@ -181,10 +296,9 @@
     end
 
-
     sprintf logfile "%s/dist.%s.%s.log" $OUTDIR $DIST_ID $COMPONENT
-    stdout $logfile
-    stderr $logfile
-
-    $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile
+#    stdout $logfile
+#    stderr $logfile
+
+    $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $DIST_STAGE --stage_id $DIST_STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile
 
     add_standard_args run
@@ -212,5 +326,4 @@
 end
 
-
 task	       dist.advance.load
   host         local
@@ -221,8 +334,7 @@
   npending     1
 
-  stdout NULL
-  stderr $LOGDIR/dist.advance.load.log
-
   task.exec
+  #  stderr $LOGSUBDIR/dist.advance.load.log
+
     $run = disttool -toadvance
     if ($DB:n == 0)
@@ -263,4 +375,5 @@
 end
 
+
 task	       dist.advance.run
   periods      -poll $RUNPOLL
@@ -270,6 +383,10 @@
   task.exec
     book npages distToAdvance -var N
-    if ($N == 0) break
     if ($NETWORK == 0) break
+    if ($N == 0) 
+    	periods -exec 10 
+	break
+    end
+    periods -exec .05
     
     # look for new components to process (pantaskState == INIT)
@@ -281,5 +398,5 @@
     book getword distToAdvance $pageName stage   -var STAGE
     book getword distToAdvance $pageName stage_id -var STAGE_ID
-    book getword distToAdvance $pageName outdir -var OUTDIR
+    book getword distToAdvance $pageName outdir -var OUTDIR_TEMPLATE
     book getword distToAdvance $pageName clean -var CLEAN
     book getword distToAdvance $pageName dbname -var DBNAME
@@ -291,7 +408,15 @@
     host anyhost
 
+    # using $DIST_ID as the "component" works fine here since we only look
+    # at the last two digits
+    set.dist.workdir.by.component $STAGE_ID "exposure" $OUTDIR_TEMPLATE OUTDIR 
+    if ("$OUTDIR" == "NULL")
+        echo ERROR failed to set workdir for $DIST_ID
+        break
+    end
+
     sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID
-    stdout $logfile
-    stderr $logfile
+#    stdout $logfile
+#    stderr $logfile
 
     $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile
@@ -320,63 +445,49 @@
 end
 
-task	       dist.queueruns
-#  host         local
-
-  periods      -poll $RUNPOLL
-  periods      -exec 30
-  periods      -timeout 45
+task dist.revert
+  host         local
+
+  periods      -poll 5.0
+  periods      -exec 60.0
+  periods      -timeout 120.0
   npending     1
-
   active false
-
-#  stdout $LOGDIR/dist.queuruns
-#  stderr $LOGDIR/dist.queueruns
+  
+  stdout NULL
+  stderr $LOGDIR/revert.log
 
   task.exec
-    $MYARGS = ""
-    # assume that we need magic unless we are running simtest
-    if ($?SIMTEST_CAMERA != 0)
-        $MYARGS = --no_magic
-    end
-
-    $run = dist_queue_runs.pl $MYARGS --stage_limit 16 --logfile $LOGDIR/dist.queueruns
-
+    if ($LABEL:n == 0) break
+    $run = disttool -revertcomponent
     if ($DB:n == 0)
-       $DBNAME = DEFAULT
+      option DEFAULT
     else
-      # save the DB name for add_standard_args
-      $DBNAME =  $DB:$distQueue_DB
-      $distQueue_DB ++
-      if ($distQueue_DB >= $DB:n) set distQueue_DB = 0
-    end
-
-    host anyhost
-
-    add_standard_args run
-
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
+      # save the DB name for the exit tasks
+      option $DB:$dist_revert_DB
+      $run = $run -dbname $DB:$dist_revert_DB
+      $dist_revert_DB ++
+      if ($dist_revert_DB >= $DB:n) set dist_revert_DB = 0
+    end
+    add_poll_labels run
+    echo $run
     command $run
   end
 
-  task.exit     $EXIT_SUCCESS
-    # nothing to do
-  end
-
-  # default exit status
+  # success
+  task.exit    0
+  end
+
+  # locked list
   task.exit    default
     showcommand failure
   end
 
-  # operation timed out?
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
   task.exit    timeout
     showcommand timeout
   end
-
-  # operation timed out?
-  task.exit    crash
-    showcommand crash
-  end
-end
+end