Index: branches/eam_branches/ipp-20101205/ippTasks/Makefile.am
===================================================================
--- branches/eam_branches/ipp-20101205/ippTasks/Makefile.am	(revision 30102)
+++ branches/eam_branches/ipp-20101205/ippTasks/Makefile.am	(revision 30446)
@@ -17,4 +17,5 @@
 	warp.pro \
 	magic.pro \
+	magic.cleanup.pro \
 	destreak.pro \
 	destreak.cleanup.pro \
Index: branches/eam_branches/ipp-20101205/ippTasks/addstar.pro
===================================================================
--- branches/eam_branches/ipp-20101205/ippTasks/addstar.pro	(revision 30102)
+++ branches/eam_branches/ipp-20101205/ippTasks/addstar.pro	(revision 30446)
@@ -173,5 +173,6 @@
     $run = $run --minidvodb_group $MINIDVODB_GROUP
 	if (("$MINIDVODB_NAME" != "NULL") && ("$MINIDVODB_NAME" != "(null)"))
-           $run = $run --minidvodb_name $MINIDVODB_NAME #have addstar_run.pl grab the 'active' name if it is NULL
+           $run = $run --minidvodb_name $MINIDVODB_NAME 
+    #have addstar_run.pl grab the 'active' name if it is NULL
 	end
     end 
Index: branches/eam_branches/ipp-20101205/ippTasks/diff.pro
===================================================================
--- branches/eam_branches/ipp-20101205/ippTasks/diff.pro	(revision 30102)
+++ branches/eam_branches/ipp-20101205/ippTasks/diff.pro	(revision 30446)
@@ -45,5 +45,5 @@
   end
   task diff.revert
-    active false
+    active true
   end
   task diff.summary.load
@@ -448,5 +448,4 @@
   periods      -timeout 120.0
   npending     1
-  active false
 
   stdout NULL
@@ -457,7 +456,11 @@
     # Only revert failures with fault=2 (SYS_ERROR), which tend to be
     # temporary filesystem problems.  Every other fault type is
-    # interesting and should be kept for debugging (and so it doesn't
+    # interesting and should be kept for debugging (and so it does not
     # continue to occur).
-    $run = difftool -revertdiffskyfile -fault 2
+    # XXX: actually we are getting enough jobs with fault 4 that work after reverting
+    # that we are going to remove that cut for now
+    # 
+    #$run = difftool -revertdiffskyfile -fault 2
+    $run = difftool -revertdiffskyfile
     if ($DB:n == 0)
       option DEFAULT
Index: branches/eam_branches/ipp-20101205/ippTasks/dist.pro
===================================================================
--- branches/eam_branches/ipp-20101205/ippTasks/dist.pro	(revision 30102)
+++ branches/eam_branches/ipp-20101205/ippTasks/dist.pro	(revision 30446)
@@ -61,5 +61,5 @@
   end
   task dist.revert
-    active false
+    active true
   end
 end
Index: branches/eam_branches/ipp-20101205/ippTasks/ipphosts.mhpcc.config
===================================================================
--- branches/eam_branches/ipp-20101205/ippTasks/ipphosts.mhpcc.config	(revision 30102)
+++ branches/eam_branches/ipp-20101205/ippTasks/ipphosts.mhpcc.config	(revision 30446)
@@ -22,5 +22,5 @@
   sky09 STR  ipp014
   sky10 STR  ipp015
-  sky11 STR  ipp016 
+  sky11 STR  ipp006 
   sky12 STR  ipp017
   sky13 STR  ipp018
@@ -125,5 +125,5 @@
   XY26  STR  ipp027
   XY27  STR  ipp028
-  XY30  STR  ipp016
+  XY30  STR  ipp006
   XY31  STR  ipp044
 
@@ -204,5 +204,5 @@
   ota26  STR  ipp027
   ota27  STR  ipp028
-  ota30  STR  ipp016
+  ota30  STR  ipp006
   ota31  STR  ipp044
 	            
@@ -298,5 +298,5 @@
   XY11  STR  ipp039
 	           
-  XY12  STR  ipp009
+  XY12  STR  ipp005
   XY13  STR  ipp024
   XY14  STR  ipp010
@@ -305,15 +305,15 @@
   XY16  STR  ipp011
   XY17  STR  ipp026
-  XY20  STR  ipp012
+  XY20  STR  ipp006
   XY21  STR  ipp041
 	           
   XY22  STR  ipp013
   XY23  STR  ipp042
-  XY24  STR  ipp014
+  XY24  STR  ipp007
   XY25  STR  ipp043
 	           
   XY26  STR  ipp027
   XY27  STR  ipp028
-  XY30  STR  ipp016
+  XY30  STR  ipp006
   XY31  STR  ipp044
 	           
@@ -358,5 +358,5 @@
   XY72  STR  ipp052
 	           
-  XY73  STR  ipp053
+  XY73  STR  ipp007
   XY74  STR  ipp015
   XY75  STR  ipp025
@@ -371,12 +371,12 @@
   sky02 STR  ipp007
   sky03 STR  ipp008
-  sky04 STR  ipp009
+  sky04 STR  ipp005
   sky05 STR  ipp010
   sky06 STR  ipp011
-  sky07 STR  ipp012
+  sky07 STR  ipp006
   sky08 STR  ipp013
-  sky09 STR  ipp014
+  sky09 STR  ipp007
   sky10 STR  ipp015
-  sky11 STR  ipp016 
+  sky11 STR  ipp006 
   sky12 STR  ipp017
   sky13 STR  ipp018
@@ -414,5 +414,5 @@
   sky45 STR  ipp051
   sky46 STR  ipp052
-  sky47 STR  ipp053
+  sky47 STR  ipp007
   sky48 STR  ipp023
   sky49 STR  ipp024
@@ -445,4 +445,4 @@
   sky76 STR  ipp051
   sky77 STR  ipp052
-  sky78 STR  ipp053
-END
+  sky78 STR  ipp006
+END
Index: branches/eam_branches/ipp-20101205/ippTasks/magic.cleanup.pro
===================================================================
--- branches/eam_branches/ipp-20101205/ippTasks/magic.cleanup.pro	(revision 30446)
+++ branches/eam_branches/ipp-20101205/ippTasks/magic.cleanup.pro	(revision 30446)
@@ -0,0 +1,220 @@
+## magic.cleanup.pro : support for the cleaning magic temporary files : -*- sh -*-
+
+# test for required global variables
+check.globals
+
+$LOGSUBDIR = $LOGDIR/magic
+mkdir $LOGSUBDIR
+
+### Initialise the books containing the tasks to do
+book init magicToCleanup
+
+### Database lists
+$magicToCleanup_DB = 0
+
+### Check status of tasks
+macro magic.cleanup.status
+  book listbook magicToCleanup
+end
+
+### Reset tasks
+macro magic.cleanup.reset
+  book init magicToCleanup
+end
+
+### Turn tasks on
+macro magic.cleanup.revert.on
+  task magic.cleanup.revert
+    active true
+  end
+end
+
+macro magic.cleanup.on
+  task magic.cleanup.load
+    active true
+  end
+  task magic.cleanup.run
+    active true
+  end
+end
+
+macro magic.cleanup.revert.off
+  task magic.cleanup.revert
+    active false
+  end
+end
+
+### Turn tasks off
+macro magic.cleanup.off
+  task magic.cleanup.load
+    active false
+  end
+  task magic.cleanup.run
+    active false
+  end
+end
+
+task	       magic.cleanup.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/magic.cleanup.load.log
+
+  task.exec
+    $run = magictool -tocleanup
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$magicToCleanup_DB
+      $run = $run -dbname $DB:$magicToCleanup_DB
+      $magicToCleanup_DB ++
+      if ($magicToCleanup_DB >= $DB:n) set magicToCleanup_DB = 0
+    end
+    add_poll_args run
+    # XXX: add this back in once our inital cleanup is done
+#    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout magicToCleanup -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook magicToCleanup
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup magicToCleanup
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task	       magic.cleanup.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 300
+
+  task.exec
+    book npages magicToCleanup -var N
+    if ($N == 0) 
+        periods -exec $RUNEXEC
+        break
+    end
+    if ($NETWORK == 0) break
+    
+    book getpage magicToCleanup 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword magicToCleanup $pageName pantaskState RUN
+    book getword magicToCleanup $pageName magic_id -var MAGIC_ID
+    book getword magicToCleanup $pageName exp_id -var EXP_ID
+    book getword magicToCleanup $pageName camera -var CAMERA
+    book getword magicToCleanup $pageName workdir -var WORKDIR
+    book getword magicToCleanup $pageName dbname -var DBNAME
+
+    host anyhost
+
+    sprintf outroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID
+
+    sprintf logfile "%s.cleanup.log" $outroot
+
+    stdout $logfile
+    stderr $logfile
+
+    $run = magic_cleanup.pl --magic_id $MAGIC_ID --exp_id $EXP_ID --camera $CAMERA --outroot $outroot --logfile $logfile
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit magicToCleanup $options:0 $JOB_STATUS
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword magicToCleanup $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword magicToCleanup $options:0 pantaskState TIMEOUT
+  end
+end
+
+task magic.cleanup.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+  active false
+
+  stdout NULL
+  stderr $LOGSUBDIR/revertnode.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = magictool -revertnode
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$magicRevertNode_DB
+      $run = $run -dbname $DB:$magicRevertNode_DB
+      $magicRevertNode_DB ++
+      if ($magicRevertNode_DB >= $DB:n) set magicRevertNode_DB = 0
+    end
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
Index: branches/eam_branches/ipp-20101205/ippTasks/nightly_science.pro
===================================================================
--- branches/eam_branches/ipp-20101205/ippTasks/nightly_science.pro	(revision 30102)
+++ branches/eam_branches/ipp-20101205/ippTasks/nightly_science.pro	(revision 30446)
@@ -7,8 +7,4 @@
     ns.detrends.off
     ns.dqstats.on
-#    ns.sweetspot.on
-#    ns.registration.on
-#    ns.burntool.on
-#    ns.chips.on
     ns.stacks.on
     ns.diffs.on
@@ -19,8 +15,4 @@
     ns.detrends.off
     ns.dqstats.off
-#    ns.sweetspot.off
-#    ns.registration.off
-#    ns.burntool.off
-#    ns.chips.off
     ns.stacks.off
     ns.diffs.off
@@ -53,34 +45,4 @@
 end
 
-# macro ns.sweetspot.on
-#   task ns.sweetspot.load
-#     active true
-#   end
-# end
-
-# macro ns.registration.on
-#   task ns.registration.load
-#     active true
-#   end
-# end
-
-# macro ns.burntool.on
-#   task ns.burntool.load
-#     active true
-#   end
-#   task ns.burntool.run
-#     active true
-#   end
-# end
-
-# macro ns.chips.on
-#   task ns.chips.load
-#     active true
-#   end
-#   task ns.chips.run
-#     active true
-#   end
-# end
-
 macro ns.stacks.on
   task ns.stacks.load
@@ -91,5 +53,5 @@
   end
   task ns.stacks.confirm
-    active true
+    active false
   end
 end
@@ -122,34 +84,4 @@
 end
 
-# macro ns.sweetspot.off
-#   task ns.sweetspot.load
-#     active false
-#   end
-# end
-
-# macro ns.registration.off
-#   task ns.registration.load
-#     active false
-#   end
-# end
-
-# macro ns.burntool.off
-#   task ns.burntool.load
-#     active false
-#   end
-#   task ns.burntool.run
-#     active false
-#   end
-# end
-
-# macro ns.chips.off
-#   task ns.chips.load
-#     active false
-#   end
-#   task ns.chips.run
-#     active false
-#   end
-# end
-
 macro ns.stacks.off
   task ns.stacks.load
@@ -174,10 +106,17 @@
 end
 
-# $ns_regPAGE = 0
-# $ns_burnPAGE = 0
-# $ns_RburnPAGE = 0
-# $ns_RburnCELL = 0
-# $ns_chipPAGE = 0
-# $ns_RchipPAGE = 0
+macro ns.stacks.confirm.on
+  task ns.stacks.confirm
+    active true
+  end
+end
+
+macro ns.stacks.confirm.off
+  task ns.stacks.confirm
+    active false
+  end
+end
+
+
 $ns_stackPAGE = 0
 $ns_RstackPAGE = 0
@@ -187,12 +126,26 @@
 
 book init nsData
+book init nsDiffs
+book init nsStacks
 #
 # Macros to control the book.
 #
 macro ns.add.date
+  if ($0 != 2) 
+    echo "USAGE: ns.add.date (YYYY-MM-DD)"
+    break
+  end
+
    book newpage nsData $1
    book setword nsData $1 nsState NEW
-   book setword nsData $1 nsStackState TOWARP
-   book setword nsData $1 nsDiffState TOWARP
+   book setword nsData $1 dbname $DB:0 
+
+   book newpage nsDiffs $1
+   book setword nsDiffs $1 nsDiffState TOWARP
+   book setword nsDiffs $1 nsObservingState UNKNOWN
+
+   book newpage nsStacks $1
+   book setword nsStacks $1 nsStackState TOWARP
+
 end
 
@@ -202,7 +155,7 @@
        book getpage nsData $i -var date
        book getword nsData $date nsState -var state
-       book getword nsData $date nsStackState -var Sstate
-       book getword nsData $date nsDiffState -var Dstate
-       book getword nsData $date nsObservingState -var Ostate
+       book getword nsStacks $date nsStackState -var Sstate
+       book getword nsDiffs $date nsDiffState -var Dstate
+       book getword nsDiffs $date nsObservingState -var Ostate
        echo $date $state $Sstate $Dstate $Ostate
     end
@@ -216,5 +169,4 @@
     book setword nsData $1 nsState $2
 end
-
 
 
@@ -236,11 +188,13 @@
     book newpage nsData $today
     book setword nsData $today nsState NEW
-    book setword nsData $today nsStackState TOWARP
-    book setword nsData $today nsDiffState TOWARP
+    book newpage nsStacks $today
+    book setword nsStacks $today nsStackState TOWARP
+    book newpage nsDiffs $today
+    book setword nsDiffs $today nsDiffState TOWARP
     book setword nsData $today dbname $DB:0 
 
 #    command echo $today
    $run = nightly_science.pl --clean_old --date $today --dbname $DB:0
-    add_standard_args $run
+#    add_standard_args run
     command $run
   end
@@ -279,5 +233,5 @@
 
     $run = nightly_science.pl --queue_detrends --date $today --dbname $DB:0
-     add_standard_args $run
+#     add_standard_args $run
      command $run
   end
@@ -317,5 +271,5 @@
 
     $run = nightly_science.pl --queue_dqstats --date $today --dbname $DB:0
-     add_standard_args $run
+#     add_standard_args $run
      command $run
 
@@ -342,4 +296,5 @@
 #
 task              ns.stacks.load
+  active          false
   host            local
   periods         -poll $LOADPOLL
@@ -353,6 +308,7 @@
 
      book getpage nsData $ns_stackPAGE -var date
-     book getword nsData $date nsStackState -var ns_STATE
-     book getword nsData $date dbname -var $DBNAME
+     book getword nsStacks $date nsStackState -var ns_STATE
+     book getword nsDiffs $date nsDiffState -var ns_diff_STATE
+     book getword nsData $date dbname -var DBNAME
      book npages nsData -var Npages
 
@@ -367,6 +323,6 @@
      if ("$ns_STATE" != "TOWARP") break
 
-     $run = nightly_science.pl --check_stacks --date $date
-     add_standard_args $run
+     $run = nightly_science.pl --check_stacks --date $date --dbname $DBNAME
+#     add_standard_args $run
      command $run
 
@@ -376,12 +332,12 @@
   task.exit   0
 #    convert 'stdout' to book format
-    book delpage nsData $options:0
-    ipptool2book stdout nsData -uniq -key date
-
-    book getword nsData $options:0 nsNmacros -var ns_Nmacros
+    book delpage nsStacks $options:0
+    ipptool2book stdout nsStacks -uniq -key date
+
+    book getword nsStacks $options:0 nsNmacros -var ns_Nmacros
     if ("$ns_Nmacros" != "NULL") 
 	for i 0 $ns_Nmacros
 	    sprintf macroName "ns%dMacro" $i
-	    book getword nsData $options:0 $macroName -var macroCmd
+	    book getword nsStacks $options:0 $macroName -var macroCmd
 	    $macroCmd
 	end
@@ -422,6 +378,7 @@
 
      book getpage nsData $ns_RstackPAGE -var date
-     book getword nsData $date nsStackState -var ns_STATE
-     book getword nsData $date dbname -var $DBNAME
+     book getword nsStacks $date nsStackState -var ns_STATE
+     book getword nsDiffs $date nsDiffState -var ns_diff_STATE
+     book getword nsData $date dbname -var DBNAME
      book npages nsData -var Npages
 
@@ -434,7 +391,7 @@
      option $date
 
-     if (("$ns_STATE" != "QUEUESTACKS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break
-     $run = nightly_science.pl --queue_stacks --date $date
-     add_standard_args $run
+     if (("$ns_STATE" != "STACKING")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break
+     $run = nightly_science.pl --queue_stacks --date $date --dbname $DBNAME
+#     add_standard_args $run
      command $run
    end
@@ -442,13 +399,13 @@
   task.exit   0
 #    convert 'stdout' to book format
-    book delpage nsData $options:0
+    book delpage nsStacks $options:0
 # We've queued up stacking. We're done with this date, so don't reload the page.
-    ipptool2book stdout nsData -uniq -key date
-
-    book getword nsData $options:0 nsNmacros -var ns_Nmacros
+    ipptool2book stdout nsStacks -uniq -key date
+
+    book getword nsStacks $options:0 nsNmacros -var ns_Nmacros
     if ("$ns_Nmacros" != "NULL") 
 	for i 0 $ns_Nmacros
 	    sprintf macroName "ns%dMacro" $i
-	    book getword nsData $options:0 $macroName -var macroCmd
+	    book getword nsStacks $options:0 $macroName -var macroCmd
 	    $macroCmd
 	end
@@ -477,4 +434,5 @@
 #
 task              ns.stacks.confirm
+  active          false
   host            local
   periods         -poll $LOADPOLL
@@ -488,6 +446,6 @@
 
      book getpage nsData $ns_CstackPAGE -var date
-     book getword nsData $date nsStackState -var ns_STATE
-     book getword nsData $date dbname -var $DBNAME
+     book getword nsStacks $date nsStackState -var ns_STATE
+     book getword nsData $date dbname -var DBNAME
      book npages nsData -var Npages
 
@@ -501,6 +459,6 @@
 
      if ("$ns_STATE" != "STACKING") break
-     $run = nightly_science.pl --confirm_stacks --date $date
-     add_standard_args $run
+     $run = nightly_science.pl --confirm_stacks --date $date --dbname $DBNAME
+#     add_standard_args $run
      command $run
    end
@@ -508,12 +466,12 @@
   task.exit   0
 #    convert 'stdout' to book format
-    book delpage nsData $options:0
-    ipptool2book stdout nsData -uniq -key date
-
-    book getword nsData $options:0 nsNmacros -var ns_Nmacros
+    book delpage nsStacks $options:0
+    ipptool2book stdout nsStacks -uniq -key date
+
+    book getword nsStacks $options:0 nsNmacros -var ns_Nmacros
     if ("$ns_Nmacros" != "NULL") 
 	for i 0 $ns_Nmacros
 	    sprintf macroName "ns%dMacro" $i
-	    book getword nsData $options:0 $macroName -var macroCmd
+	    book getword nsStacks $options:0 $macroName -var macroCmd
 	    $macroCmd
 	end
@@ -543,4 +501,5 @@
 #
 task              ns.diffs.load
+  active          false
   host            local
   periods         -poll $LOADPOLL
@@ -554,6 +513,7 @@
 
      book getpage nsData $ns_diffPAGE -var date
-     book getword nsData $date nsDiffState -var ns_STATE
-     book getword nsData $date dbname -var $DBNAME
+     book getword nsDiffs $date nsDiffState -var ns_STATE
+     book getword nsStacks $date nsStackState -var ns_stack_STATE
+     book getword nsData $date dbname -var DBNAME
      book npages nsData -var Npages
 
@@ -567,6 +527,6 @@
 
      if ("$ns_STATE" != "TOWARP") break
-     $run = nightly_science.pl --check_diffs --date $date
-     add_standard_args $run
+     $run = nightly_science.pl --check_diffs --date $date --dbname $DBNAME
+#     add_standard_args $run
      command $run
    end
@@ -575,12 +535,12 @@
   task.exit   0
 #    convert 'stdout' to book format
-    book delpage nsData $options:0
-    ipptool2book stdout nsData -uniq -key date
-
-    book getword nsData $options:0 nsNmacros -var ns_Nmacros
+    book delpage nsDiffs $options:0
+    ipptool2book stdout nsDiffs -uniq -key date
+
+    book getword nsDiffs $options:0 nsNmacros -var ns_Nmacros
     if ("$ns_Nmacros" != "NULL") 
 	for i 0 $ns_Nmacros
 	    sprintf macroName "ns%dMacro" $i
-	    book getword nsData $options:0 $macroName -var macroCmd
+	    book getword nsDiffs $options:0 $macroName -var macroCmd
 	    $macroCmd
 	end
@@ -613,5 +573,5 @@
   periods         -poll $LOADPOLL
   periods         -exec $LOADEXEC
-  periods         -timeout 120
+  periods         -timeout 900
   npending        1
 
@@ -621,6 +581,7 @@
 
      book getpage nsData $ns_RdiffPAGE -var date
-     book getword nsData $date nsDiffState -var ns_STATE
-     book getword nsData $date dbname -var $DBNAME
+     book getword nsDiffs $date nsDiffState -var ns_STATE
+     book getword nsStacks $date nsStackState -var ns_stack_STATE
+     book getword nsData $date dbname -var DBNAME
      book npages nsData -var Npages
 
@@ -633,7 +594,7 @@
      option $date
 
-     if (("$ns_STATE" != "QUEUEDIFFS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break
-     $run = nightly_science.pl --queue_diffs --date $date
-     add_standard_args $run
+     if (("$ns_STATE" != "DIFFING")&&("$ns_STATE" != "QUEUEDIFFS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break
+     $run = nightly_science.pl --queue_diffs --date $date --dbname $DBNAME
+#     add_standard_args $run
      command $run
    end
@@ -641,13 +602,13 @@
   task.exit   0
 #    convert 'stdout' to book format
-    book delpage nsData $options:0
+    book delpage nsDiffs $options:0
 # We've queued up diffing. We're done with this date, so don't reload the page.
-    ipptool2book stdout nsData -uniq -key date
-
-    book getword nsData $options:0 nsNmacros -var ns_Nmacros
+    ipptool2book stdout nsDiffs -uniq -key date
+
+    book getword nsDiffs $options:0 nsNmacros -var ns_Nmacros
     if ("$ns_Nmacros" != "NULL") 
 	for i 0 $ns_Nmacros
 	    sprintf macroName "ns%dMacro" $i
-	    book getword nsData $options:0 $macroName -var macroCmd
+	    book getword nsDiffs $options:0 $macroName -var macroCmd
 	    $macroCmd
 	end
Index: branches/eam_branches/ipp-20101205/ippTasks/pstamp.pro
===================================================================
--- branches/eam_branches/ipp-20101205/ippTasks/pstamp.pro	(revision 30102)
+++ branches/eam_branches/ipp-20101205/ippTasks/pstamp.pro	(revision 30446)
@@ -258,4 +258,5 @@
         book getword pstampRequest $pageName ds_outProduct -var PRODUCT
         book getword pstampRequest $pageName outdir -var OUTDIR
+        book getword pstampRequest $pageName need_magic -var NEED_MAGIC
         book getword pstampRequest $pageName label -var LABEL
 
@@ -263,4 +264,8 @@
 
         $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --outdir $OUTDIR --label $LABEL --redirect-output
+
+        if ($NEED_MAGIC != 0)
+            $run = $run --need_magic
+        end
 
         add_standard_args run
Index: branches/eam_branches/ipp-20101205/ippTasks/publish.pro
===================================================================
--- branches/eam_branches/ipp-20101205/ippTasks/publish.pro	(revision 30102)
+++ branches/eam_branches/ipp-20101205/ippTasks/publish.pro	(revision 30446)
@@ -79,4 +79,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -131,4 +132,5 @@
     book getword publishRun $pageName dbname -var DBNAME
     book getword publishRun $pageName output_format -var OUTPUT_FORMAT
+    book getword publishRun $pageName need_magic -var NEED_MAGIC
 
     stdout $LOGDIR/publish.run.log
@@ -139,4 +141,9 @@
 
     $run = publish_file.pl --pub_id $PUB_ID --camera $CAMERA --workdir $WORKDIR --product $PRODUCT --stage $STAGE --stage_id $STAGE_ID --output_format $OUTPUT_FORMAT --redirect-output
+
+    if ("$NEED_MAGIC" == "T") 
+        $run = $run --need-magic
+    end
+
     add_standard_args run
 
Index: branches/eam_branches/ipp-20101205/ippTasks/register.pro
===================================================================
--- branches/eam_branches/ipp-20101205/ippTasks/register.pro	(revision 30102)
+++ branches/eam_branches/ipp-20101205/ippTasks/register.pro	(revision 30446)
@@ -9,4 +9,5 @@
 book init regPendingExp
 book init regPendingBurntoolImfile
+book init regDates
 
 macro register.reset
@@ -14,4 +15,5 @@
   book init regPendingExp
   book init regPendingBurntoolImfile
+  book init regDates
 end
 
@@ -21,4 +23,5 @@
   book listbook regPendingExp
   book listbook regPendingBurntoolImfile
+  book listbook regDates
 end
 
@@ -30,4 +33,7 @@
     active true
   end
+  task register.imfile.revert
+    active false
+  end
   task register.exp.load
     active true
@@ -51,4 +57,7 @@
     active false
   end
+  task register.imfile.revert
+    active false
+  end
   task register.exp.load
     active false
@@ -65,5 +74,49 @@
 end
 
+macro register.revert.on
+  task register.imfile.revert
+    active true
+  end
+end
+
+macro register.revert.off
+  task register.imfile.revert
+    active false
+  end
+end
+
+macro burntool.on
+  task register.burntool.load
+    active true
+  end
+  task register.burntool.run
+    active true
+  end
+end
+
+macro burntool.off
+  task register.burntool.load
+    active false
+  end
+  task register.burntool.run
+    active false
+  end
+end
+
+macro register.add.date
+  if ($0 != 2) 
+    echo "USAGE: register.add.date (YYYY-MM-DD)"
+    break
+  end
+
+   book newpage regDates $1
+   book setword regDates $1 nsState NEW
+   book setword regDates $1 dbname $DB:0 
+end
+
+
 $valid_burntool_value = 14
+$sunrise = 17:30:00
+$sunset  = 03:30:00
 
 macro replace.survey.values
@@ -147,4 +200,5 @@
 $regPendingImfile_DB = 0
 $regPendingBurntoolImfile_DB = 0
+$regRevertImfile_DB = 0
 $regPendingExp_DB = 0
 
@@ -265,4 +319,5 @@
     # XXX register_imfile.pl differs from the standard script : it does not have an 'outroot' argument, and it does not take '--redirect'
     $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI --logfile $logfile --bytes $BYTES --md5sum $MD5SUM
+    $run = $run --sunset $sunset --sunrise $sunrise
     add_standard_args run
 
@@ -295,4 +350,48 @@
   end
 end
+
+task register.imfile.revert
+  host         local
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout       NULL
+  stderr       $LOGDIR/register.imfile.revert.log
+
+  task.exec
+    $run = regtool -revertprocessedimfile
+    if ($DB:n == 0) 
+      option DEFAULT
+    else
+      option $DB:$regRevertImfile_DB
+      $run = $run -dbname $DB:$regRevertImfile_DB
+      $regRevertImfile_DB ++
+      if ($regRevertImfile_DB >= $DB:n) set regRevertImfile_DB = 0
+    end
+    add_poll_args run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
 
 # select exposures ready for register_exp.pl
@@ -463,9 +562,20 @@
   # select entried from the current DB; cycle to the next DB, if it exists
   task.exec
+     book npages regDates -var Npages
+     if ($Npages == 0) 
+       $today = `date -u +%Y-%m-%d`
+     else 
+       book getpage regDates $reg_datePAGE -var today
+       $reg_datePAGE ++
+       if ($reg_datePAGE >= $Npages) set reg_datePAGE = 0
+     end
+
     $run = regtool -pendingburntoolimfile
-    $today = `date -u +%Y-%m-%d`
+
 # debugging purposes
-#    $today = "2010-12-06"
-    $run = $run -date $today -valid_burntool $valid_burntool_value
+#   $today = "2010-12-25"
+    $dateobs_begin = $today\T$sunset
+    $dateobs_end   = $today\T$sunrise
+    $run = $run -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end -valid_burntool $valid_burntool_value
     if ($DB:n == 0)
       option DEFAULT
@@ -574,5 +684,37 @@
 end
 
-    
+task   register.initday.load
+  host            local
+  periods         -poll $LOADPOLL
+  periods         -exec $LOADEXEC
+  periods         -timeout 30
+  trange          0:00:00 1:00:00 -nmax 1
+  npending        1
+
+  task.exec
+    $today = `date -u +%Y-%m-%d`
+    book newpage regDates $today
+    book setword regDates $today nsState NEW
+    book setword regDates $today dbname $DB:0
+
+    command true
+  end
+
+  task.exit       0
+    # nothing to do here?
+  end
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
+  end
+  #operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
    
   
Index: branches/eam_branches/ipp-20101205/ippTasks/survey.pro
===================================================================
--- branches/eam_branches/ipp-20101205/ippTasks/survey.pro	(revision 30102)
+++ branches/eam_branches/ipp-20101205/ippTasks/survey.pro	(revision 30446)
@@ -259,10 +259,11 @@
 # user functions to manipulate destreak labels
 macro survey.add.destreak
-  if ($0 != 3)
-    echo "USAGE: survey.add.destreak (label) (workdir base)"
+  if ($0 != 4)
+    echo "USAGE: survey.add.destreak (label) (workdir base) (recovery root)"
     break
   end
   book newpage SURVEY_DESTREAK $1
   book setword SURVEY_DESTREAK $1 WORKDIR $2
+  book setword SURVEY_DESTREAK $1 RECOVERYROOT $3
   book setword SURVEY_DESTREAK $1 STATE PENDING
 end
@@ -366,10 +367,12 @@
 # user functions to manipulate publish labels
 macro survey.add.publish
-  if ($0 != 3)
-    echo "USAGE: survey.add.publish (label) (client_id)"
+  if ($0 != 5)
+    echo "USAGE: survey.add.publish (tag) (label) (client_id) (comment)"
     break
   end
   book newpage SURVEY_PUBLISH $1
-  book setword SURVEY_PUBLISH $1 CLIENT_ID $2
+  book setword SURVEY_PUBLISH $1 LABEL $2
+  book setword SURVEY_PUBLISH $1 CLIENT_ID $3
+  book setword SURVEY_PUBLISH $1 COMMENT $4
   book setword SURVEY_PUBLISH $1 STATE PENDING
 end
@@ -821,6 +824,10 @@
     book setword SURVEY_DESTREAK $label STATE DONE
     book getword SURVEY_DESTREAK $label WORKDIR -var workdir
+    book getword SURVEY_DESTREAK $label RECOVERYROOT -var recoveryroot
   
     $run = magic_destreak_defineruns.pl --label $label --workdir $workdir/$label
+    if ("$recoveryroot" != "NULL") 
+        $run = $run --recoveryroot $recoveryroot/$label
+    end
 
     if ($DB:n == 0)
@@ -1084,13 +1091,15 @@
     endif
 
-    book getpage SURVEY_PUBLISH 0 -var label -key STATE NEW
-    if ("$label" == "NULL")
-      # All labels have been done --- reset
-      # echo "Resetting labels"
+    # survey.publish allows multiple entries per label.
+    # The key is called tag and must be unique
+    book getpage SURVEY_PUBLISH 0 -var tag -key STATE NEW
+    if ("$tag" == "NULL")
+      # All tags have been done --- reset
+      # echo "Resetting tags"
       for i 0 $N
-        book getpage SURVEY_PUBLISH $i -var label
-	book setword SURVEY_PUBLISH $label STATE NEW
+        book getpage SURVEY_PUBLISH $i -var tag
+	book setword SURVEY_PUBLISH $tag STATE NEW
       end
-      book getpage SURVEY_PUBLISH 0 -var label -key STATE NEW
+      book getpage SURVEY_PUBLISH 0 -var tag -key STATE NEW
 
       # Select different database
@@ -1099,8 +1108,14 @@
     end
 
-    book setword SURVEY_PUBLISH $label STATE DONE
-    book getword SURVEY_PUBLISH $label CLIENT_ID -var client_id
+    book setword SURVEY_PUBLISH $tag STATE DONE
+    book getword SURVEY_PUBLISH $tag LABEL -var label
+    book getword SURVEY_PUBLISH $tag CLIENT_ID -var client_id
+    book getword SURVEY_PUBLISH $tag COMMENT -var comment
   
     $run = pubtool -definerun -label $label -client_id $client_id
+
+    if ("$comment" != "NULL") 
+        $run = $run -comment $comment
+    end
 
     if ($DB:n == 0)
@@ -1111,6 +1126,8 @@
     end
     
-    #echo $run
+    echo $run
+
     command $run
+    
   end