Index: branches/eam_branches/ipp-20100823/ippTasks/Makefile.am
===================================================================
--- branches/eam_branches/ipp-20100823/ippTasks/Makefile.am	(revision 29124)
+++ branches/eam_branches/ipp-20100823/ippTasks/Makefile.am	(revision 29515)
@@ -18,4 +18,5 @@
 	magic.pro \
 	destreak.pro \
+	destreak.cleanup.pro \
 	diff.pro \
 	stack.pro \
Index: branches/eam_branches/ipp-20100823/ippTasks/destreak.cleanup.pro
===================================================================
--- branches/eam_branches/ipp-20100823/ippTasks/destreak.cleanup.pro	(revision 29515)
+++ branches/eam_branches/ipp-20100823/ippTasks/destreak.cleanup.pro	(revision 29515)
@@ -0,0 +1,172 @@
+## destreak.cleanup.pro : tasks for the cleaning up destreak stage : -*- sh -*-
+
+# test for required global variables
+check.globals
+
+$LOGSUBDIR = $LOGDIR/destreak
+mkdir $LOGSUBDIR
+
+### Initialise the books containing the tasks to do
+book init magicDSToCleanup
+
+### indexes into Database lists
+$magicDSToCleanup_DB = 0
+
+### Check status of tasks
+macro destreak.cleanup.status
+    book listbook magicDSToCleanup
+end
+
+### Reset tasks
+macro destreak.cleanup.reset
+    book init magicDSToCleanup
+end
+
+### Turn tasks on
+macro destreak.cleanup.on
+    task destreak.cleanup.load
+        active true
+    end
+    task destreak.cleanup.run
+        active true
+    end
+end
+
+### Turn tasks off
+macro destreak.cleanup.off
+    task destreak.cleanup.load
+        active false
+    end
+    task destreak.cleanup.run
+        active false
+    end
+end
+
+task	       destreak.cleanup.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 20
+  npending     1
+  active       false
+
+  stdout NULL
+  stderr $LOGSUBDIR/destreak.cleanup.load.log
+
+  task.exec
+    $run = magicdstool -tocleanup
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+
+      # save the DB name for the exit tasks
+      option $DB:$magicDSToCleanup_DB
+      $run = $run -dbname $DB:$magicDSToCleanup_DB
+
+      # only bump database number after we have gone through all of the stages
+      $magicDSToCleanup_DB ++
+      if ($magicDSToCleanup_DB >= $DB:n) set magicDSToCleanup_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout magicDSToCleanup -key magic_ds_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook magicDSToCleanup
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup magicDSToCleanup
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task	       destreak.cleanup.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+  active       false
+
+  task.exec
+    stdout $LOGSUBDIR/destreak.cleanup.run.log
+    stderr $LOGSUBDIR/destreak.cleanup.run.log
+
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $RUNEXEC
+
+    book npages magicDSToCleanup -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new images (pantaskState == INIT)
+    book getpage magicDSToCleanup 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword magicDSToCleanup $pageName pantaskState RUN
+    book getword magicDSToCleanup $pageName magic_ds_id -var MAGIC_DS_ID
+    book getword magicDSToCleanup $pageName camera -var CAMERA
+    book getword magicDSToCleanup $pageName stage -var STAGE
+    book getword magicDSToCleanup $pageName outroot -var OUTROOT
+    book getword magicDSToCleanup $pageName dbname -var DBNAME
+
+    sprintf logfile "%s/mds.%s.cleanup.log" $OUTROOT $MAGIC_DS_ID
+
+    host anyhost
+
+    $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --stage $STAGE --camera $CAMERA --logfile $logfile
+
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    # if we are ready to run, drop the retry timeout low so we fill up the queue
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit    0
+    process_exit magicDSToCleanup $options:0 $JOB_STATUS
+   end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+    process_exit magicDSToCleanup $options:0 $JOB_STATUS
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword magicDSToCleanup $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword magicDSToCleanup $options:0 pantaskState TIMEOUT
+  end
+end
Index: branches/eam_branches/ipp-20100823/ippTasks/destreak.pro
===================================================================
--- branches/eam_branches/ipp-20100823/ippTasks/destreak.pro	(revision 29124)
+++ branches/eam_branches/ipp-20100823/ippTasks/destreak.pro	(revision 29515)
@@ -10,5 +10,4 @@
 book init magicToDS
 book init magicDSToRevert
-book init magicDSToCleanup
 
 ### indexes into Database lists
@@ -30,13 +29,17 @@
 $magicDSRevertStage = 0
 
+macro destreak.show.stages
+    echo $DS_STAGE:n stages enabled for destreak processing
+    for i 0 $DS_STAGE:n
+        echo $i $DS_STAGE:$i
+    end
+    echo current magicDSStage = $magicDSStage
+    echo current magicDSRevertStage = $magicDSRevertStage
+end
+
 ### Check status of tasks
 macro destreak.status
-    # since echo and listbook output go to different files these do not come in order
-    echo magicToDS
     book listbook magicToDS
-    echo magicDSToRevert
     book listbook magicDSToRevert
-    echo magicDSToCleanup
-    book listbook magicDSToCleanup
 end
 
@@ -45,5 +48,4 @@
     book init magicToDS
     book init magicDSToRevert
-    book init magicDSToCleanup
 end
 
@@ -73,13 +75,4 @@
 end
 
-macro destreak.cleanup.on
-    task destreak.cleanup.load
-        active true
-    end
-    task destreak.cleanup.run
-        active true
-    end
-end
-
 ### Turn tasks off
 macro destreak.off
@@ -106,13 +99,4 @@
     end
 end
-macro destreak.cleanup.off
-    task destreak.cleanup.load
-        active false
-    end
-    task destreak.cleanup.run
-        active false
-    end
-end
-
 
 task	       destreak.load
@@ -203,5 +187,7 @@
     book getword magicToDS $pageName camera -var CAMERA
     book getword magicToDS $pageName streaks_uri -var STREAKS
+    book getword magicToDS $pageName streaks_path_base -var STREAKS_PATH_BASE
     book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS
+    book getword magicToDS $pageName inv_streaks_path_base -var INV_STREAKS_PATH_BASE
     book getword magicToDS $pageName stage -var STAGE
     book getword magicToDS $pageName stage_id -var STAGE_ID
@@ -229,5 +215,5 @@
     # TODO: do not add recoveryroot or replace if they are null or zero
 
-    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED
+    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED
 
     add_standard_args run
@@ -274,7 +260,6 @@
 
   periods      -poll $LOADPOLL
-  #periods      -exec $LOADEXEC
   periods      -exec 30
-  periods      -timeout 60
+  periods      -timeout 300
   npending     1
 
@@ -293,4 +278,6 @@
     add_poll_args run
     add_poll_labels run
+    # since this command can be expensive, reduce the limit
+    $run = $run -limit 24
     command $run
   end
@@ -317,8 +304,7 @@
 task	       destreak.revert.load
   host         local
-
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 20
+  periods      -poll 60.0
+  periods      -exec 120.
+  periods      -timeout 120
   npending     1
   active       false
@@ -518,131 +504,2 @@
 end
 
-task	       destreak.cleanup.load
-  host         local
-
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 20
-  npending     1
-  active       false
-
-  stdout NULL
-  stderr $LOGSUBDIR/destreak.cleanup.load.log
-
-  task.exec
-    $run = magicdstool -tocleanup
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-
-      # save the DB name for the exit tasks
-      option $DB:$magicDSToCleanup_DB
-      $run = $run -dbname $DB:$magicDSToCleanup_DB
-
-      # only bump database number after we have gone through all of the stages
-      $magicDSToCleanup_DB ++
-      if ($magicDSToCleanup_DB >= $DB:n) set magicDSToCleanup_DB = 0
-    end
-    add_poll_args run
-    add_poll_labels run
-    command $run
-  end
-
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout magicDSToCleanup -key magic_ds_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook magicDSToCleanup
-    end
-
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup magicDSToCleanup
-  end
-
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-
-  task.exit    crash
-    showcommand crash
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-
-task	       destreak.cleanup.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       false
-
-  task.exec
-    stdout $LOGSUBDIR/destreak.cleanup.run.log
-    stderr $LOGSUBDIR/destreak.cleanup.run.log
-
-    # if we are unable to run the 'exec', use a long retry time
-    periods -exec $RUNEXEC
-
-    book npages magicDSToCleanup -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    
-    # look for new images (pantaskState == INIT)
-    book getpage magicDSToCleanup 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-
-    book setword magicDSToCleanup $pageName pantaskState RUN
-    book getword magicDSToCleanup $pageName magic_ds_id -var MAGIC_DS_ID
-    book getword magicDSToCleanup $pageName camera -var CAMERA
-    book getword magicDSToCleanup $pageName stage -var STAGE
-    book getword magicDSToCleanup $pageName outroot -var OUTROOT
-    book getword magicDSToCleanup $pageName dbname -var DBNAME
-
-    sprintf logfile "%s/mds.%s.cleanup.log" $OUTROOT $MAGIC_DS_ID
-
-    host anyhost
-
-    $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --stage $STAGE --camera $CAMERA --logfile $logfile
-
-    add_standard_args run
-
-    # save the pageName for future reference below
-    options $pageName
-
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    # if we are ready to run, drop the retry timeout low so we fill up the queue
-    periods -exec 0.05
-    command $run
-  end
-
-  # default exit status
-  task.exit    0
-    process_exit magicDSToCleanup $options:0 $JOB_STATUS
-   end
-
-  # locked list
-  task.exit    default
-    showcommand failure
-    process_exit magicDSToCleanup $options:0 $JOB_STATUS
-  end
-
-  task.exit    crash
-    showcommand crash
-    book setword magicDSToCleanup $options:0 pantaskState CRASH
-  end
-
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword magicDSToCleanup $options:0 pantaskState TIMEOUT
-  end
-end
-
Index: branches/eam_branches/ipp-20100823/ippTasks/diskbalance.pro
===================================================================
--- branches/eam_branches/ipp-20100823/ippTasks/diskbalance.pro	(revision 29515)
+++ branches/eam_branches/ipp-20100823/ippTasks/diskbalance.pro	(revision 29515)
@@ -0,0 +1,229 @@
+## diskbalance.pro : tasks for data replication : -*- sh -*-
+## this file contains the tasks for maintaining duplicates as needed in Nebulous
+## these tasks use the books balancePending
+
+# test for required global variables
+check.globals
+
+# load in the local configuration (NEB_USER, etc.)
+module nebulous.site.pro
+
+$LOGSUBDIR = $LOGDIR/balance
+mkdir $LOGSUBDIR
+
+book init balancePending
+book init balanceControl
+
+macro balance.reset
+  book init balancePending
+end
+
+macro balance.status
+  book listbook balancePending
+end
+
+macro balance.on
+  task balance.govern
+    active true
+  end
+  task balance.load
+    active true
+  end
+  task balance.run
+    active true
+  end
+end
+
+macro balance.off
+  task balance.govern
+    active false
+  end
+  task balance.load
+    active false
+  end
+  task balance.run
+    active false
+  end
+end
+
+macro set.host.for.balance
+  if ($0 != 2)
+    echo "USAGE: set.host.for.balance (hostname)"
+    break
+  end
+
+  if (not($PARALLEL))
+    host local
+    return
+  end
+
+# parse volume name
+
+  if ("$1" == "NULL")
+    host anyhost
+  else
+    host $1
+  end
+end
+
+macro set.balance.numbers
+  if ($0 != 3)
+    echo "USAGE: set.balance.numbers (N_sources) (M_destinations)"
+    break
+  end
+
+  book newpage balanceControl control
+  book setword balanceControl control nSources $1
+  book setword balanceControl control mDestinations $2
+end
+
+# the balance process interacts with only the single Nebulous server
+
+# Each 'pendingbalance query is limited to a finite number of so_id values.
+# Each time we call balance.load, we increment the so_id start by the range value
+# If the pendingbalance query exits with exit status 10, we start over at 0
+
+$BALANCE_SO_ID_START = 0
+$BALANCE_SO_ID_RANGE = 500000
+
+# Select Nebulous objects which should be shuffled
+task           balance.load
+  host         local
+
+  periods      -poll 0.5
+  periods      -exec 5
+  periods      -timeout 1500
+  npending     1
+
+  # logs
+  stdout NULL
+  stderr $LOGSUBDIR/balance.log
+    
+  task.exec
+    book npages balancePending -var N
+    if ($N > 2000)
+      process_cleanup balancePending
+      break
+    end
+
+    book getword balanceControl control nSources -var BALANCE_N
+    book getword balanceControl control mDestinations -var BALANCE_M
+
+    command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingbalance --limit 500 --so_id_start $BALANCE_SO_ID_START --so_id_range $BALANCE_SO_ID_RANGE  --balancesources $BALANCE_N --balancedestination $BALANCE_M
+
+  end
+
+   # success : 0 -- we did not hit the limit, advance so_id counter
+  task.exit 0
+    # advance the so_id counter
+    $BALANCE_SO_ID_START = $BALANCE_SO_ID_START + $BALANCE_SO_ID_RANGE
+
+    # convert 'stdout' to book format
+    ipptool2book stdout balancePending -key key -uniq -setword pantaskState INIT
+
+    if ($VERBOSE > 2)
+      book listbook balancePending
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup balancePending
+  end
+
+  # success : 1 -- we DID hit the limit, do NOT advance so_id counter 
+  task.exit 1
+    # convert 'stdout' to book format
+    ipptool2book stdout balancePending -key key -uniq -setword pantaskState INIT
+
+    if ($VERBOSE > 2)
+      book listbook balancePending
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup balancePending
+  end
+
+  # out of so_id range, reset
+  task.exit 10
+    # advance the so_id counter
+    $BALANCE_SO_ID_START = 0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# task to execute shuffles
+task            balance.run
+  periods       -poll 0.5
+  periods       -exec 5
+  periods       -timeout 30
+
+  task.exec
+    book npages balancePending -var N
+    if ($NETWORK == 0) break
+    if ($N == 0)
+        periods -exec 5
+        break
+    end
+    periods -exec 0.05
+
+    # look for new objects in balancePending
+    book getpage balancePending 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setwork balancePending $pageName pantaskState RUN
+
+    book getword balancePending $pageName key              -var KEY
+    book getword balancePending $pageName source_name      -var SOURCE
+    book getword balancePending $pageName destination_name -var DESTINATION
+    
+    # Fix this with multihost restriction when possible.
+    set.host.for.balance $DESTINATION
+
+    stdout NULL
+    stderr $LOGSUBDIR/balance.log
+
+    $run = neb-shift --volume $DESTINATION $KEY $SOURCE
+    
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+	echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit default
+    process_exit balancePending $options:0 $JOB_STATUS
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword balancePending $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword balancePending $options:0 pantaskState TIMEOUT
+  end
+end
+
+
+
+    
+
Index: branches/eam_branches/ipp-20100823/ippTasks/dist.pro
===================================================================
--- branches/eam_branches/ipp-20100823/ippTasks/dist.pro	(revision 29124)
+++ branches/eam_branches/ipp-20100823/ippTasks/dist.pro	(revision 29515)
@@ -176,7 +176,7 @@
   host         local
 
-  periods      -poll $LOADPOLL
+  periods      -poll 20
   periods      -exec $LOADEXEC
-  periods      -timeout 30
+  periods      -timeout 300
   npending     1
 
@@ -338,6 +338,6 @@
 
   periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
+  periods      -exec 30
+  periods      -timeout 3000
   npending     1
 
@@ -479,5 +479,4 @@
     end
     add_poll_labels run
-    echo $run
     command $run
   end
Index: branches/eam_branches/ipp-20100823/ippTasks/magic.pro
===================================================================
--- branches/eam_branches/ipp-20100823/ippTasks/magic.pro	(revision 29124)
+++ branches/eam_branches/ipp-20100823/ippTasks/magic.pro	(revision 29515)
@@ -288,4 +288,6 @@
     book getword magicToProcess $pageName camera -var CAMERA
     book getword magicToProcess $pageName workdir -var WORKDIR_TEMPLATE
+    book getword magicToProcess $pageName raw_workdir -var RAW_TEMPLATE
+    book getword magicToProcess $pageName exp_name -var EXP_NAME
     book getword magicToProcess $pageName dbname -var DBNAME
 
@@ -308,4 +310,15 @@
     sprintf baseroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID
 
+    if ("$NODE" == "root")
+        # This is the root node, put the streaks file into the rawExps workdir
+        # which is presumably in nebulous and thus will be replicatable
+        set.workdir.by.camera $CAMERA FPA $RAW_TEMPLATE $default_host RAW_WORKDIR
+        sprintf final_outroot "%s/%s.%s/%s.%s.mgc.%s" $RAW_WORKDIR $EXP_NAME $EXP_ID $EXP_NAME $EXP_ID $MAGIC_ID
+
+        $EXTRA_ARGS = --final-outroot $final_outroot
+    else
+        $EXTRA_ARGS = ""
+    end
+
     $logfile = $baseroot.$NODE.log
     if ("$logfile" == "") 
@@ -319,5 +332,5 @@
     mkdir $outpath
 
-    $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile
+    $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile $EXTRA_ARGS
     add_standard_args run
 
Index: branches/eam_branches/ipp-20100823/ippTasks/nightly_stacks.pro
===================================================================
--- branches/eam_branches/ipp-20100823/ippTasks/nightly_stacks.pro	(revision 29124)
+++ branches/eam_branches/ipp-20100823/ippTasks/nightly_stacks.pro	(revision 29515)
@@ -436,8 +436,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
@@ -498,8 +500,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
@@ -560,8 +564,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
@@ -623,8 +629,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
@@ -684,8 +692,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
@@ -747,8 +757,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
@@ -870,8 +882,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
Index: branches/eam_branches/ipp-20100823/ippTasks/pstamp.pro
===================================================================
--- branches/eam_branches/ipp-20100823/ippTasks/pstamp.pro	(revision 29124)
+++ branches/eam_branches/ipp-20100823/ippTasks/pstamp.pro	(revision 29515)
@@ -14,5 +14,7 @@
 $pstampRev_DB = 0
 $pstampDep_DB = 0
+$pstampRevDep_DB = 0
 $pstampCleanup_DB = 0
+$pstampStopFaulted_DB = 0
 
 # set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config
@@ -89,7 +91,4 @@
         active false
     end
-    task pstamp.job.revert
-        active false
-    end
     task pstamp.dependent.load
         active false
@@ -104,7 +103,19 @@
         active true
     end
+    task pstamp.dependent.revert
+        active true
+    end
+    task pstamp.stopfaulted
+        active true
+    end
 end
 macro pstamp.revert.off
     task pstamp.job.revert
+        active false
+    end
+    task pstamp.dependent.revert
+        active false
+    end
+    task pstamp.stopfaulted
         active false
     end
@@ -513,5 +524,5 @@
         stdout NULL
         stderr $LOGSUBDIR/pstamp.job.revert.log
-        $run = pstamptool -revertjob -all
+        $run = pstamptool -revertjob
         if ($DB:n == 0)
             option DEFAULT
@@ -798,2 +809,83 @@
 end
 
+task pstamp.dependent.revert
+    host        local
+
+    periods     -poll $LOADPOLL
+    periods     -exec 900
+    periods     -timeout 300
+    npending    1
+
+    task.exec
+        stdout NULL
+        stderr $LOGSUBDIR/pstamp.dependent.revert.log
+        $run = pstamptool -revertdependent
+        if ($DB:n == 0)
+            option DEFAULT
+        else 
+            $run = $run $PS_DBSERVER -dbname $DB:$pstampRevDep_DB
+            $pstampRevDep_DB ++
+            if ($pstampRevDep_DB >= $DB:n) set pstampRevDep_DB = 0
+        end
+        add_poll_args run
+        add_poll_labels run
+        command $run
+    end
+
+    task.exit $EXIT_SUCCESS
+        # nothing to do
+    end
+
+    task.exit   default
+        showcommand failure
+    end
+
+    task.exit   crash
+        showcommand crash
+    end
+
+    task.exit   timeout
+        showcommand timeout
+    end
+end
+
+task pstamp.stopfaulted
+    host        local
+
+    periods     -poll $LOADPOLL
+    periods     -exec 30
+    periods     -timeout 300
+    npending    1
+
+    task.exec
+        stderr $LOGSUBDIR/pstamp.stopfaulted.log
+        stderr $LOGSUBDIR/pstamp.stopfaulted.log
+        $run = pstampstopfaulted
+        if ($DB:n == 0)
+            option DEFAULT
+        else 
+            $run = $run $PS_DBSERVER -dbname $DB:$pstampStopFaulted_DB
+            $pstampStopFaulted_DB ++
+            if ($pstampStopFaulted_DB >= $DB:n) set pstampStopFaulted_DB = 0
+        end
+        add_poll_args run
+        add_poll_labels run
+        command $run
+    end
+
+    task.exit $EXIT_SUCCESS
+        # nothing to do
+    end
+
+    task.exit   default
+        showcommand failure
+    end
+
+    task.exit   crash
+        showcommand crash
+    end
+
+    task.exit   timeout
+        showcommand timeout
+    end
+end
Index: branches/eam_branches/ipp-20100823/ippTasks/publish.pro
===================================================================
--- branches/eam_branches/ipp-20100823/ippTasks/publish.pro	(revision 29124)
+++ branches/eam_branches/ipp-20100823/ippTasks/publish.pro	(revision 29515)
@@ -75,4 +75,6 @@
 
   task.exec
+    if ($LABEL:n == 0) break
+    #otherwise everything will be queued when there is no label.
     $run = pubtool -definerun
     if ($DB:n == 0)
Index: branches/eam_branches/ipp-20100823/ippTasks/replicate.pro
===================================================================
--- branches/eam_branches/ipp-20100823/ippTasks/replicate.pro	(revision 29124)
+++ branches/eam_branches/ipp-20100823/ippTasks/replicate.pro	(revision 29515)
@@ -173,4 +173,5 @@
     book getword replicatePending $pageName volume_name -var VOLUME_NAME
     book getword replicatePending $pageName volume_host -var VOLUME_HOST
+    book getword replicatePending $pageName command     -var COMMAND
 
     set.host.for.replicate $VOLUME_HOST
@@ -180,5 +181,5 @@
 
     # these operations do not require a database to be specified
-    $run = neb-replicate --copies $NEED_COPIES $KEY
+    $run = $COMMAND $KEY
 
     # save the pageName for future reference below
Index: branches/eam_branches/ipp-20100823/ippTasks/science.cleanup.pro
===================================================================
--- branches/eam_branches/ipp-20100823/ippTasks/science.cleanup.pro	(revision 29124)
+++ branches/eam_branches/ipp-20100823/ippTasks/science.cleanup.pro	(revision 29515)
@@ -113,5 +113,8 @@
   task.exec
     book npages chipPendingCleanup -var N
-    if ($N == 0) break
+    if ($N == 0) 
+        periods -exec $RUNEXEC
+        break
+    end
     if ($NETWORK == 0) break
     
@@ -143,4 +146,5 @@
       echo command $run
     end
+    periods -exec 0.05
     command $run
   end
@@ -569,5 +573,8 @@
   task.exec
     book npages warpPendingCleanup -var N
-    if ($N == 0) break
+    if ($N == 0) 
+        periods -exec $RUNEXEC
+        break
+    end
     if ($NETWORK == 0) break
     
@@ -599,4 +606,5 @@
       echo command $run
     end
+    periods -exec 0.05
     command $run
   end
@@ -721,5 +729,8 @@
   task.exec
     book npages diffCleanup -var N
-    if ($N == 0) break
+    if ($N == 0) 
+        periods -exec $RUNEXEC
+        break
+    end
     if ($NETWORK == 0) break
     
@@ -751,4 +762,5 @@
       echo command $run
     end
+    periods -exec 0.05
     command $run
   end