Index: trunk/ippTasks/detrend.cleanup.pro
===================================================================
--- trunk/ippTasks/detrend.cleanup.pro	(revision 27515)
+++ trunk/ippTasks/detrend.cleanup.pro	(revision 27516)
@@ -6,77 +6,39 @@
 check.globals
 
-book init detCleanupProcessedImfile
-book init detCleanupProcessedExp
+book init detCleanupProcessed
+book init detCleanupResid
+
 book init detCleanupStackedImfile
 book init detCleanupNormStatImfile
 book init detCleanupNormImfile
 book init detCleanupNormExp
-book init detCleanupResidImfile
-book init detCleanupResidExp
+
 
 macro detclean.reset
-  book init detCleanupProcessedImfile
-  book init detCleanupProcessedExp
+  book init detCleanupProcessed
+  book init detCleanupResid
+
   book init detCleanupStackedImfile
-
   book init detCleanupNormStatImfile
   book init detCleanupNormImfile
   book init detCleanupNormExp
-
-  book init detCleanupResidImfile
-  book init detCleanupResidExp
 end
 
 macro detclean.status
-  echo detCleanupProcessedImfile
-  book listbook detCleanupProcessedImfile
-  echo detCleanupProcessedExp
-  book listbook detCleanupProcessedExp
-  echo detCleanupStackedImfile
+  book listbook detCleanupProcessed
+  book listbook detCleanupResid
+
   book listbook detCleanupStackedImfile
-
   book listbook detCleanupNormStatImfile
   book listbook detCleanupNormImfile
   book listbook detCleanupNormExp
-  book listbook detCleanupResidImfile
-  book listbook detCleanupResidExp
+
 end
 
 macro detclean.on
-  task detrend.cleanup.process.load
+  task detrend.cleanup.processed.load
     active true
   end
-  task detrend.cleanup.process.run
-    active true
-  end
-  task detrend.cleanup.processexp.load
-    active true
-  end
-  task detrend.cleanup.processexp.run
-    active true
-  end
-  task detrend.cleanup.stack.load
-    active true
-  end
-  task detrend.cleanup.stack.run
-    active true
-  end
-
-  task detrend.cleanup.norm.load
-    active true
-  end
-  task detrend.cleanup.norm.run
-    active true
-  end
-  task detrend.cleanup.normexp.load
-    active true
-  end
-  task detrend.cleanup.normexp.run
-    active true
-  end
-  task detrend.cleanup.normstat.load
-    active true
-  end
-  task detrend.cleanup.normstat.run
+  task detrend.cleanup.processed.run
     active true
   end
@@ -88,25 +50,46 @@
     active true
   end
-  task detrend.cleanup.residexp.load
-    active true
-  end
-  task detrend.cleanup.residexp.run
-    active true
-  end
+
+#  task detrend.cleanup.stack.load
+#    active true
+#  end
+#  task detrend.cleanup.stack.run
+#    active true
+#  end
+#  task detrend.cleanup.norm.load
+#    active true
+#  end
+#  task detrend.cleanup.norm.run
+#    active true
+#  end
+#  task detrend.cleanup.normexp.load
+#    active true
+#  end
+#  task detrend.cleanup.normexp.run
+#    active true
+#  end
+#  task detrend.cleanup.normstat.load
+#    active true
+#  end
+#  task detrend.cleanup.normstat.run
+#    active true
+#  end
 end
 
 macro detclean.off
-  task detrend.cleanup.process.load
-    active false
-  end
-  task detrend.cleanup.process.run
-    active false
-  end
-  task detrend.cleanup.processexp.load
-    active false
-  end
-  task detrend.cleanup.processexp.run
-    active false
-  end
+  task detrend.cleanup.processed.load
+    active false
+  end
+  task detrend.cleanup.processed.run
+    active false
+  end
+
+  task detrend.cleanup.resid.load
+    active false
+  end
+  task detrend.cleanup.resid.run
+    active false
+  end
+
   task detrend.cleanup.stack.load
     active false
@@ -133,31 +116,19 @@
     active false
   end
-  task detrend.cleanup.resid.load
-    active false
-  end
-  task detrend.cleanup.resid.run
-    active false
-  end
-  task detrend.cleanup.residexp.load
-    active false
-  end
-  task detrend.cleanup.residexp.run
-    active false
-  end
+
 end
 
 
 # these variables will cycle through the known database names
-$detCleanupProcessedImfile_DB = 0
-$detCleanupProcessedExp_DB = 0
+$detCleanupProcessed_DB = 0
+$detCleanupResid_DB = 0
+
 $detCleanupStackedImfile_DB = 0
 $detCleanupNormStatImfile_DB = 0
 $detCleanupNormImfile_DB = 0
 $detCleanupNormExp_DB = 0
-$detCleanupResidImfile_DB = 0
-$detCleanupResidExp_DB = 0
-
-######## cleanup process imfile ########
-task	       detrend.cleanup.process.load
+
+######## cleanup processed imfile/exp ########
+task	       detrend.cleanup.processed.load
   host         local
 
@@ -169,16 +140,16 @@
 
   stdout NULL
-  stderr $LOGDIR/detrend.cleanup.process.imfile.log
-
-  task.exec
-    $run = dettool -pendingcleanup_processedimfile
+  stderr $LOGDIR/detrend.cleanup.processed.log
+
+  task.exec
+    $run = dettool -pendingcleanup_processedexp 
     if ($DB:n == 0)
       option DEFAULT
     else
       # save the DB name for the exit tasks
-      option $DB:$detCleanupProcessedImfile_DB
-      $run = $run -dbname $DB:$detCleanupProcessedImfile_DB
-      $detCleanupProcessedImfile_DB ++
-      if ($detCleanupProcessedImfile_DB >= $DB:n) set detCleanupProcessedImfile_DB = 0
+      option $DB:$detCleanupProcessed_DB
+      $run = $run -dbname $DB:$detCleanupProcessed_DB
+      $detCleanupProcessed_DB ++
+      if ($detCleanupProcessed_DB >= $DB:n) set detCleanupProcessed_DB = 0
     end
     add_poll_args run
@@ -189,11 +160,11 @@
   task.exit    0
     # convert 'stdout' to book format
-    ipptool2book stdout detCleanupProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    ipptool2book stdout detCleanupProcessed -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     if ($VERBOSE > 2)
-      book listbook detCleanupProcessedImfile
+      book listbook detCleanupProcessed
     end
 
     # delete existing entries in the appropriate pantaskStates
-    process_cleanup detCleanupProcessedImfile
+    process_cleanup detCleanupProcessed
   end
 
@@ -214,5 +185,5 @@
 
 # run the ipp_cleanup.pl script on pending images
-task	       detrend.cleanup.process.run
+task	       detrend.cleanup.processed.run
   periods      -poll $RUNPOLL
   periods      -exec $RUNEXEC
@@ -221,28 +192,27 @@
 
   task.exec
-    book npages detCleanupProcessedImfile -var N
+    book npages detCleanupProcessed -var N
     if ($N == 0) break
     if ($NETWORK == 0) break
     
-    # look for new images in detCleanupProcessedImfile (pantaskState == INIT)
-    book getpage detCleanupProcessedImfile 0 -var pageName -key pantaskState INIT
+    # look for new images in detCleanupProcessed (pantaskState == INIT)
+    book getpage detCleanupProcessed 0 -var pageName -key pantaskState INIT
     if ("$pageName" == "NULL") break
 
-    book setword detCleanupProcessedImfile $pageName pantaskState RUN
-    book getword detCleanupProcessedImfile $pageName det_id   -var DET_ID   
-    book getword detCleanupProcessedImfile $pageName exp_id   -var EXP_ID   
-    book getword detCleanupProcessedImfile $pageName class_id -var CLASS_ID 
-    book getword detCleanupProcessedImfile $pageName camera   -var CAMERA
-    book getword detCleanupProcessedImfile $pageName data_state    -var CLEANUP_MODE
-    book getword detCleanupProcessedImfile $pageName dbname   -var DBNAME
-
-    # specify choice of local or remote host based on camera and chip (class_id)
+    book setword detCleanupProcessed $pageName pantaskState RUN
+    book getword detCleanupProcessed $pageName det_id   -var DET_ID   
+    book getword detCleanupProcessed $pageName exp_id   -var EXP_ID   
+    book getword detCleanupProcessed $pageName camera   -var CAMERA
+    book getword detCleanupProcessed $pageName data_state    -var CLEANUP_MODE
+    book getword detCleanupProcessed $pageName dbname   -var DBNAME
+
+    # specify choice of local or remote host based on camera and diff (class_id)
     set.host.for.camera $CAMERA FPA
 
-    stdout $LOGDIR/detrend.cleanup.process.imfile.log
-    stderr $LOGDIR/detrend.cleanup.process.imfile.log
+    stdout $LOGDIR/detrend.cleanup.processed.log
+    stderr $LOGDIR/detrend.cleanup.processed.log
 
     # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage detrend.processed --stage_id $DET_ID.$EXP_ID --camera $CAMERA --mode $CLEANUP_MODE --dbname $DBNAME
     add_standard_args run
 
@@ -259,5 +229,5 @@
   # default exit status
   task.exit    default
-    process_exit detCleanupProcessedImfile $options:0 $JOB_STATUS
+    process_exit detCleanupProcessed $options:0 $JOB_STATUS
   end
 
@@ -266,5 +236,5 @@
     showcommand crash
     echo "hostname: $JOB_HOSTNAME"
-    book setword detCleanupProcessedImfile $options:0 pantaskState CRASH
+    book setword detCleanupProcessed $options:0 pantaskState CRASH
   end
 
@@ -272,11 +242,12 @@
   task.exit    timeout
     showcommand timeout
-    book setword detCleanupProcessedImfile $options:0 pantaskState TIMEOUT
-  end
-end
- 
-
-######## cleanup process exp ########
-task	       detrend.cleanup.processexp.load
+    book setword detCleanupProcessed $options:0 pantaskState TIMEOUT
+  end
+end
+
+
+
+######## cleanup resid imfile/exp ########
+task	       detrend.cleanup.resid.load
   host         local
 
@@ -288,16 +259,16 @@
 
   stdout NULL
-  stderr $LOGDIR/detrend.cleanup.process.exp.log
-
-  task.exec
-    $run = dettool -pendingcleanup_processedexp
+  stderr $LOGDIR/detrend.cleanup.resid.log
+
+  task.exec
+    $run = dettool -pendingcleanup_residexp
     if ($DB:n == 0)
       option DEFAULT
     else
       # save the DB name for the exit tasks
-      option $DB:$detCleanupProcessedExp_DB
-      $run = $run -dbname $DB:$detCleanupProcessedExp_DB
-      $detCleanupProcessedExp_DB ++
-      if ($detCleanupProcessedExp_DB >= $DB:n) set detCleanupProcessedExp_DB = 0
+      option $DB:$detCleanupResid_DB
+      $run = $run -dbname $DB:$detCleanupResid_DB
+      $detCleanupResid_DB ++
+      if ($detCleanupResid_DB >= $DB:n) set detCleanupResid_DB = 0
     end
     add_poll_args run
@@ -308,11 +279,11 @@
   task.exit    0
     # convert 'stdout' to book format
-    ipptool2book stdout detCleanupProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    ipptool2book stdout detCleanupResid -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     if ($VERBOSE > 2)
-      book listbook detCleanupProcessedExp
+      book listbook detCleanupResid
     end
 
     # delete existing entries in the appropriate pantaskStates
-    process_cleanup detCleanupProcessedExp
+    process_cleanup detCleanupResid
   end
 
@@ -333,5 +304,5 @@
 
 # run the ipp_cleanup.pl script on pending images
-task	       detrend.cleanup.processexp.run
+task	       detrend.cleanup.resid.run
   periods      -poll $RUNPOLL
   periods      -exec $RUNEXEC
@@ -340,27 +311,28 @@
 
   task.exec
-    book npages detCleanupProcessedExp -var N
+    book npages detCleanupResid -var N
     if ($N == 0) break
     if ($NETWORK == 0) break
     
-    # look for new images in detCleanupProcessedExp (pantaskState == INIT)
-    book getpage detCleanupProcessedExp 0 -var pageName -key pantaskState INIT
+    # look for new images in detCleanupResid (pantaskState == INIT)
+    book getpage detCleanupResid 0 -var pageName -key pantaskState INIT
     if ("$pageName" == "NULL") break
 
-    book setword detCleanupProcessedExp $pageName pantaskState RUN
-    book getword detCleanupProcessedExp $pageName det_id   -var DET_ID   
-    book getword detCleanupProcessedExp $pageName exp_id   -var EXP_ID   
-    book getword detCleanupProcessedExp $pageName camera -var CAMERA
-    book getword detCleanupProcessedExp $pageName data_state -var CLEANUP_MODE
-    book getword detCleanupProcessedExp $pageName dbname -var DBNAME
-
-    # specify choice of local or remote host based on camera and chip (class_id)
+    book setword detCleanupResid $pageName pantaskState RUN
+    book getword detCleanupResid $pageName det_id   -var DET_ID   
+    book getword detCleanupResid $pageName exp_id   -var EXP_ID   
+    book getword detCleanupResid $pageName iteration -var ITERATION     
+    book getword detCleanupResid $pageName camera -var CAMERA
+    book getword detCleanupResid $pageName data_state -var CLEANUP_MODE
+    book getword detCleanupResid $pageName dbname -var DBNAME
+
+    # specify choice of local or remote host based on camera and diff (class_id)
     set.host.for.camera $CAMERA FPA
 
-    stdout $LOGDIR/detrend.cleanup.process.exp.log
-    stderr $LOGDIR/detrend.cleanup.process.exp.log
+    stdout $LOGDIR/detrend.cleanup.resid.log
+    stderr $LOGDIR/detrend.cleanup.resid.log
 
     # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage detrend.resid --stage_id $DET_ID.$EXP_ID  --camera $CAMERA --mode $CLEANUP_MODE --dbname $DBNAME
     add_standard_args run
 
@@ -377,5 +349,5 @@
   # default exit status
   task.exit    default
-    process_exit detCleanupProcessedExp $options:0 $JOB_STATUS
+    process_exit detCleanupResid $options:0 $JOB_STATUS
   end
 
@@ -384,5 +356,5 @@
     showcommand crash
     echo "hostname: $JOB_HOSTNAME"
-    book setword detCleanupProcessedExp $options:0 pantaskState CRASH
+    book setword detCleanupResid $options:0 pantaskState CRASH
   end
 
@@ -390,5 +362,5 @@
   task.exit    timeout
     showcommand timeout
-    book setword detCleanupProcessedExp $options:0 pantaskState TIMEOUT
+    book setword detCleanupResid $options:0 pantaskState TIMEOUT
   end
 end
@@ -402,5 +374,5 @@
   periods      -timeout 30
   npending     1
-  active       true
+  active       false
 
   stdout NULL
@@ -449,4 +421,5 @@
 end
 
+
 # run the ipp_cleanup.pl script on pending images
 task	       detrend.cleanup.stack.run
@@ -521,5 +494,5 @@
   periods      -timeout 30
   npending     1
-  active       true
+  active       false
 
   stdout NULL
@@ -573,5 +546,5 @@
   periods      -exec $RUNEXEC
   periods      -timeout 60
-  active       true
+  active       false
 
   task.exec
@@ -636,5 +609,5 @@
   periods      -timeout 30
   npending     1
-  active       true
+  active       false
 
   stdout NULL
@@ -688,5 +661,5 @@
   periods      -exec $RUNEXEC
   periods      -timeout 60
-  active       true
+  active       false
 
   task.exec
@@ -752,5 +725,5 @@
   periods      -timeout 30
   npending     1
-  active       true
+  active       false
 
   stdout NULL
@@ -804,5 +777,5 @@
   periods      -exec $RUNEXEC
   periods      -timeout 60
-  active       true
+  active       false
 
   task.exec
@@ -860,239 +833,2 @@
  
 
-######## cleanup resid imfile ########
-task	       detrend.cleanup.resid.load
-  host         local
-
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
-  npending     1
-  active       true
-
-  stdout NULL
-  stderr $LOGDIR/detrend.cleanup.resid.imfile.log
-
-  task.exec
-    $run = dettool -pendingcleanup_residimfile
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$detCleanupResidImfile_DB
-      $run = $run -dbname $DB:$detCleanupResidImfile_DB
-      $detCleanupResidImfile_DB ++
-      if ($detCleanupResidImfile_DB >= $DB:n) set detCleanupResidImfile_DB = 0
-    end
-    add_poll_args run
-    command $run
-  end
-
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout detCleanupResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook detCleanupResidImfile
-    end
-
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup detCleanupResidImfile
-  end
-
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-
-  task.exit    crash
-    showcommand crash
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-
-# run the ipp_cleanup.pl script on pending images
-task	       detrend.cleanup.resid.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       true
-
-  task.exec
-    book npages detCleanupResidImfile -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    
-    # look for new images in detCleanupResidImfile (pantaskState == INIT)
-    book getpage detCleanupResidImfile 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-
-    book setword detCleanupResidImfile $pageName pantaskState RUN
-    book getword detCleanupResidImfile $pageName det_id   -var DET_ID   
-    book getword detCleanupResidImfile $pageName exp_id   -var EXP_ID   
-    book getword detCleanupResidImfile $pageName class_id -var CLASS_ID 
-    book getword detCleanupResidImfile $pageName iteration -var ITERATION     
-    book getword detCleanupResidImfile $pageName camera -var CAMERA
-    book getword detCleanupResidImfile $pageName data_state -var CLEANUP_MODE
-    book getword detCleanupResidImfile $pageName dbname -var DBNAME
-
-    # specify choice of local or remote host based on camera and chip (class_id)
-    set.host.for.camera $CAMERA FPA
-
-    stdout $LOGDIR/detrend.cleanup.resid.imfile.log
-    stderr $LOGDIR/detrend.cleanup.resid.imfile.log
-
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID  --camera $CAMERA --mode $CLEANUP_MODE
-    add_standard_args run
-
-    # save the pageName for future reference below
-    options $pageName
-
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    command $run
-  end
-
-  # default exit status
-  task.exit    default
-    process_exit detCleanupResidImfile $options:0 $JOB_STATUS
-  end
-
-  # locked list
-  task.exit    crash
-    showcommand crash
-    echo "hostname: $JOB_HOSTNAME"
-    book setword detCleanupResidImfile $options:0 pantaskState CRASH
-  end
-
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword detCleanupResidImfile $options:0 pantaskState TIMEOUT
-  end
-end
- 
-
-######## cleanup resid exp ########
-task	       detrend.cleanup.residexp.load
-  host         local
-
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
-  npending     1
-  active       true
-
-  stdout NULL
-  stderr $LOGDIR/detrend.cleanup.resid.exp.log
-
-  task.exec
-    $run = dettool -pendingcleanup_residexp
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$detCleanupResidExp_DB
-      $run = $run -dbname $DB:$detCleanupResidExp_DB
-      $detCleanupResidExp_DB ++
-      if ($detCleanupResidExp_DB >= $DB:n) set detCleanupResidExp_DB = 0
-    end
-    add_poll_args run
-    command $run
-  end
-
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout detCleanupResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook detCleanupResidExp
-    end
-
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup detCleanupResidExp
-  end
-
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-
-  task.exit    crash
-    showcommand crash
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-
-# run the ipp_cleanup.pl script on pending images
-task	       detrend.cleanup.residexp.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       true
-
-  task.exec
-    book npages detCleanupResidExp -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    
-    # look for new images in detCleanupResidExp (pantaskState == INIT)
-    book getpage detCleanupResidExp 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-
-    book setword detCleanupResidExp $pageName pantaskState RUN
-    book getword detCleanupResidExp $pageName det_id    -var DET_ID   
-    book getword detCleanupResidExp $pageName exp_id    -var EXP_ID   
-    book getword detCleanupResidExp $pageName iteration -var ITERATION
-    book getword detCleanupResidExp $pageName camera 	-var CAMERA
-    book getword detCleanupResidExp $pageName data_state  	-var CLEANUP_MODE
-    book getword detCleanupResidExp $pageName dbname 	-var DBNAME
-
-    # specify choice of local or remote host based on camera and chip (class_id)
-    set.host.for.camera $CAMERA FPA
-
-    stdout $LOGDIR/detrend.cleanup.resid.exp.log
-    stderr $LOGDIR/detrend.cleanup.resid.exp.log
-
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE
-    add_standard_args run
-
-    # save the pageName for future reference below
-    options $pageName
-
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    command $run
-  end
-
-  # default exit status
-  task.exit    default
-    process_exit detCleanupResidExp $options:0 $JOB_STATUS
-  end
-
-  # locked list
-  task.exit    crash
-    showcommand crash
-    echo "hostname: $JOB_HOSTNAME"
-    book setword detCleanupResidExp $options:0 pantaskState CRASH
-  end
-
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword detCleanupResidExp $options:0 pantaskState TIMEOUT
-  end
-end