Index: trunk/ippTasks/detrend.norm.pro
===================================================================
--- trunk/ippTasks/detrend.norm.pro	(revision 10745)
+++ trunk/ippTasks/detrend.norm.pro	(revision 11210)
@@ -1,12 +1,5 @@
-
-## this file contains the tasks for running the detrend normalization stage
-
-## these tasks use the queue DetrendNorm
-## the DetrendNorm queue contains:
-## STATE DET_ID ITER
-
-queueinit DetrendNorm
-queueinit DetrendNormStat
-queueinit DetrendNormExp
+## this file contains the tasks for running the detrend normalization stages
+## these tasks use the books detPendingNormStatImfile detPendingNormImfile detPendingNormExp
+
 if ($?network == 0)
   $network = 1
@@ -15,9 +8,4 @@
   $parallel = 0
 end
-
-$DetrendNormFail = 0
-$DetrendNormStatFail = 0
-$DetrendNormExpFail = 0
-
 if ($?VERBOSE == 0)
   echo "VERBOSE not defined: load pantasks.pro first"
@@ -28,18 +16,20 @@
   break
 end
+
 $LOGSUBDIR = $LOGDIR/detrend
 exec mkdir -p $LOGSUBDIR
 
+book init detPendingNormStatImfile
+book init detPendingNormImfile
+book init detPendingNormExp
+
 macro detnorm.status
-  queueprint DetrendNorm
-  queueprint DetrendNormStat
-  queueprint DetrendNormExp
-  echo "DetrendNormFail: $DetrendNormFail"
-  echo "DetrendNormStatFail: $DetrendNormStatFail"
-  echo "DetrendNormExpFail: $DetrendNormExpFail"
+  book listbook detPendingNormStatImfile
+  book listbook detPendingNormImfile
+  book listbook detPendingNormExp
 end
 
 # select images ready for copy 
-# new entries are added to queue DetrendNormStat
+# new entries are added to detPendingNormStatImfile
 # compare the new list with the ones already selected
 task	       dettool.normstat.load
@@ -50,38 +40,23 @@
   periods      -timeout 30
 
-  stdout $LOGSUBDIR/dettool.normstat.log
+  stdout NULL
   stderr $LOGSUBDIR/dettool.normstat.log
 
   task.exec
-    command      dettool -tonormalizedstat -simple -limit {$DetrendNormStatFail + 20}
+    command dettool -tonormalizedstat -limit 20
   end
 
   # success
   task.exit    0
-    # XXX is it necessary for these to be local?
-    # drop after this task macro is done?
-    local i Nqueue
-
-    # compare output with newImage queue
-    # only add entries which don't exist in queue
-    queuesize stdout -var Nqueue
-    for i 0 $Nqueue
-      queuepop stdout -var line
-      if ($VERBOSE > 2)
-        echo $line
-      end
-      list word -split $line
-      $DET_ID   = $word:0
-      $DET_TYPE = $word:1
-      $ITER     = $word:2
-      $CAMERA   = $word:3
-      queuepush DetrendNormStat -uniq -key 1:3 "NEW $DET_ID $DET_TYPE $ITER $CAMERA"
+    # convert 'stdout' to book format
+    ipptool2book stdout detPendingNormStatImfile -key det_id:iteration -uniq
+    if ($VERBOSE > 2)
+      book listbook detPendingNormStatImfile
     end
 
     # delete existing entries which are DONE
-    queuedrop DetrendNormStat -key 0 DONE
-  end
-
-  # locked list
+    book delpage detPendingNormStatImfile -key state DONE
+  end
+
   # default exit status
   task.exit    default
@@ -102,28 +77,17 @@
 
   task.exec
-    queuesize DetrendNormStat -var N
+    book npages detPendingNormStatImfile -var N
     if ($N == 0) break
     if ($network == 0) break
     
-    # look for new images on the internal DetrendNormStat queue
-    # caution with these 'if' statements: syntax errors 
-    # will make the task fail without given a good status
-    queuepop DetrendNormStat -var line -key 0 NEW
-    if ("$line" == "NULL") break
-
-    strpop line state
-    queuepush DetrendNormStat -replace -key 1:3 "RUN $line"
-
-    # the DetrendNormStat queue contains:
-    # STATE DET_ID ITER
-    list word -split $line
-    $DET_ID   = $word:0
-    $DET_TYPE = $word:1
-    $ITER     = $word:2
-    $CAMERA   = $word:3
-
-    # XXX add $WORKDIR/$LOG_DIR
-    stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
-    stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
+    # look for new images in detPendingNormStatImfile
+    book getpage detPendingNormStatImfile 0 -var pageName -key state NULL
+    if ($pageName == NULL) break
+
+    book setword detPendingNormStatImfile $pageName state RUN
+    book getword detPendingNormStatImfile $pageName det_id    -var DET_ID
+    book getword detPendingNormStatImfile $pageName det_type  -var DET_TYPE
+    book getword detPendingNormStatImfile $pageName iteration -var ITERATION
+    book getword detPendingNormStatImfile $pageName camera    -var CAMERA  
 
     # specify choice of remote host:
@@ -134,34 +98,33 @@
     end
 
+    # XXX add $WORKDIR/$LOG_DIR
+    # XXX use ipp_filename.pl to lookup output file names
+    stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log
+    stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATIION.log
+
+    # save the pageName for future reference below
+    options $pageName
+
     # create example job options as a demonstration
-    options "$line"
     if ($VERBOSE > 1)
-      echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE
-    end
-    command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE
-  end
-
-  # success
-  task.exit 0
-    queuepush DetrendNormStat -replace -key 1:3 "DONE $options:0"
-  end
-
-  # default exit status
-  task.exit    default
-    showcommand failure
-    queuepush DetrendNormStat -replace -key 1:3 "FAIL $options:0"
-    $DetrendNormStatFail ++
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-    queuepush DetrendNormStat -replace -key 1:3 "TIMEOUT $options:0"
-   $DetrendNormStatFail ++
+      echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE
+    end
+    command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit detPendingNormStatImfile $options:0 $JOB_STATUS
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword detPendingNormStatImfile $options:0 state TIMEOUT
   end
 end
 
 # select images ready for copy 
-# new entries are added to queue DetrendNorm
+# new entries are added to detPendingNormImfile
 # compare the new list with the ones already selected
 task	       dettool.norm.load
@@ -172,41 +135,23 @@
   periods      -timeout 30
 
-  stdout $LOGSUBDIR/dettool.norm.log
+  stdout NULL
   stderr $LOGSUBDIR/dettool.norm.log
 
   task.exec
-    command      dettool -tonormalize -simple -limit {$DetrendNormFail + 20}
+    command dettool -tonormalize -limit 20
   end
 
   # success
   task.exit    0
-    # XXX is it necessary for these to be local?
-    # drop after this task macro is done?
-    local i Nqueue
-
-    # compare output with newImage queue
-    # only add entries which don't exist in queue
-    queuesize stdout -var Nqueue
-    for i 0 $Nqueue
-      queuepop stdout -var line
-      if ($VERBOSE > 2)
-        echo $line
-      end
-      list word -split $line
-      $DET_TYPE = $word:0
-      $CAMERA   = $word:1
-      $URI      = $word:2
-      $DET_ID   = $word:3
-      $ITER     = $word:4
-      $CLASS_ID = $word:5
-      $NORM     = $word:6
-      queuepush DetrendNorm -uniq -key 4:5 "NEW $DET_TYPE $CAMERA $URI $DET_ID $ITER $CLASS_ID $NORM"
+    # convert 'stdout' to book format
+    ipptool2book stdout detPendingNormImfile -key det_id:iteration -uniq
+    if ($VERBOSE > 2)
+      book listbook detPendingNormImfile
     end
 
     # delete existing entries which are DONE
-    queuedrop DetrendNorm -key 0 DONE
-  end
-
-  # locked list
+    book delpage detPendingNormImfile -key state DONE
+  end
+
   # default exit status
   task.exit    default
@@ -227,30 +172,20 @@
 
   task.exec
-    queuesize DetrendNorm -var N
+    book npages detPendingNormImfile -var N
     if ($N == 0) break
     if ($network == 0) break
     
-    # look for new images on the internal DetrendNorm queue
-    # caution with these 'if' statements: syntax errors 
-    # will make the task fail without given a good status
-    queuepop DetrendNorm -var line -key 0 NEW
-    if ("$line" == "NULL") break
-
-    strpop line state
-    queuepush DetrendNorm -replace -key 4:5 "RUN $line"
-
-    # the DetrendNorm queue contains:
-    # STATE DET_ID ITER
-    list word -split $line
-    $DET_TYPE = $word:0
-    $CAMERA   = $word:1
-    $URI      = $word:2
-    $DET_ID   = $word:3
-    $ITER     = $word:4
-    $CLASS_ID = $word:5
-    $NORM     = $word:6
-
-    stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
-    stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
+    # look for new images in detPendingNormImfile
+    book getpage detPendingNormImfile 0 -var pageName -key state NULL
+    if ($pageName == NULL) break
+
+    book setword detPendingNormImfile $pageName state RUN
+    book getword detPendingNormImfile $pageName det_type  -var DET_TYPE 
+    book getword detPendingNormImfile $pageName camera    -var CAMERA   
+    book getword detPendingNormImfile $pageName uri       -var URI      
+    book getword detPendingNormImfile $pageName det_id    -var DET_ID   
+    book getword detPendingNormImfile $pageName iteration -var ITERATION     
+    book getword detPendingNormImfile $pageName class_id  -var CLASS_ID 
+    book getword detPendingNormImfile $pageName norm      -var NORM     
 
     # specify choice of remote host:
@@ -261,34 +196,32 @@
     end
 
+    # XXX use ipp_filename.pl to lookup output file names
+    stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log
+    stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log
+
+    # save the pageName for future reference below
+    options $pageName
+
     # create example job options as a demonstration
-    options "$line"
     if ($VERBOSE > 1)
-      echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE
-    end
-    command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE
-  end
-
-  # success
-  task.exit 0
-    queuepush DetrendNorm -replace -key 4:5 "DONE $options:0"
-  end
-
-  # default exit status
-  task.exit    default
-    showcommand failure
-    queuepush DetrendNorm -replace -key 4:5 "FAIL $options:0"
-    $DetrendNormFail ++
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-    queuepush DetrendNorm -replace -key 4:5 "TIMEOUT $options:0"
-    $DetrendNormFail ++
+      echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE
+    end
+    command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit detPendingNormImfile $options:0 state FAIL
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword detPendingNormImfile $options:0 state TIMEOUT
   end
 end
 
 # select images ready for copy 
-# new entries are added to queue DetrendNormExp
+# new entries are added to detPendingNormExp
 # compare the new list with the ones already selected
 task	       dettool.normexp.load
@@ -299,41 +232,23 @@
   periods      -timeout 30
 
-  stdout $LOGSUBDIR/dettool.normexp.log
+  stdout NULL
   stderr $LOGSUBDIR/dettool.normexp.log
 
   task.exec
-    command      dettool -tonormalizedexp -simple -limit {$DetrendNormExpFail + 20}
+    command dettool -tonormalizedexp -limit 20
   end
 
   # success
   task.exit    0
-    # XXX is it necessary for these to be local?
-    # drop after this task macro is done?
-    local i Nqueue
-
-    # compare output with newImage queue
-    # only add entries which don't exist in queue
-    queuesize stdout -var Nqueue
-    for i 0 $Nqueue
-      queuepop stdout -var line
-      if ($VERBOSE > 2)
-        echo $line
-      end
-      list word -split $line
-      $DET_ID   = $word:0
-      $ITER     = $word:1
-      $DET_TYPE = $word:2
-      $CAMERA   = $word:3
-      #$TELESCOPE= $word:4
-      #$EXP_TYPE = $word:5
-      #$IMFILES  = $word:6
-      queuepush DetrendNormExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $CAMERA"
+    # convert 'stdout' to book format
+    ipptool2book stdout detPendingNormExp -key det_id:iteration -uniq
+    if ($VERBOSE > 2)
+      book listbook detPendingNormExp
     end
 
     # delete existing entries which are DONE
-    queuedrop DetrendNormExp -key 0 DONE
-  end
-
-  # locked list
+    book delpage detPendingNormExp -key state DONE
+  end
+
   # default exit status
   task.exit    default
@@ -354,32 +269,17 @@
 
   task.exec
-    queuesize DetrendNormExp -var N
-   
+    book npages detPendingNormExp -var N
     if ($N == 0) break
     if ($network == 0) break
     
-    # look for new images on the internal DetrendNormExp queue
-    # caution with these 'if' statements: syntax errors 
-    # will make the task fail without given a good status
-    queuepop DetrendNormExp -var line -key 0 NEW
-    if ("$line" == "NULL") break
-
-    strpop line state
-    queuepush DetrendNormExp -replace -key 1:2 "RUN $line"
-
-    # the DetrendNormExp queue contains:
-    # STATE DET_ID ITER
-    list word -split $line
-    $DET_ID   = $word:0
-    $ITER     = $word:1
-    $DET_TYPE = $word:2
-    $CAMERA   = $word:3
-    #$TELESCOPE= $word:4
-    #$EXP_TYPE = $word:5
-    #$IMFILES  = $word:6
-
-    # XXX add $WORKDIR/$LOG_DIR
-    stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
-    stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
+    # look for new images in detPendingNormExp
+    book getpage detPendingNormExp 0 -var pageName -key state NULL
+    if ($pageName == NULL) break
+
+    book setword detPendingNormExp $pageName state RUN
+    book getword detPendingNormExp $pageName det_id    -var DET_ID  
+    book getword detPendingNormExp $pageName iteration -var ITERATION    
+    book getword detPendingNormExp $pageName det_type  -var DET_TYPE
+    book getword detPendingNormExp $pageName camera    -var CAMERA  
 
     # specify choice of remote host:
@@ -390,29 +290,27 @@
     end
 
+    # XXX add $WORKDIR/$LOG_DIR
+    stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log
+    stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log
+
+    # save the pageName for future reference below
+    options $pageName
+
     # create example job options as a demonstration
-    options "$line"
     if ($VERBOSE > 1)
-      echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE
-    end
-    command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE
-  end
-
-  # success
-  task.exit 0
-    queuepush DetrendNormExp -replace -key 1:2 "DONE $options:0"
-  end
-
-  # default exit status
-  task.exit    default
-    showcommand failure
-    queuepush DetrendNormExp -replace -key 1:2 "FAIL $options:0"
-    $DetrendNormExpFail ++
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-    queuepush DetrendNormExp -replace -key 1:2 "TIMEOUT $options:0"
-    $DetrendNormExpFail ++
-  end
-end
+      echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE
+    end
+    command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit detPendingNormExp $options:0 $JOB_STATUS
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword detPendingNormExp $options:0 state TIMEOUT
+  end
+end