Changeset 10692

Timestamp:

Dec 12, 2006, 11:58:21 PM (20 years ago)

Author:

eugene

Message:

added global logdirs, VERBOSE output, failure tracking

Location:

trunk/ippTasks

Files:

• detrend.norm.pro (modified) (26 diffs)
• detrend.process.pro (modified) (15 diffs)
• detrend.reject.pro (modified) (9 diffs)
• detrend.resid.pro (modified) (14 diffs)
• detrend.stack.pro (modified) (9 diffs)
• phase0.pro (modified) (8 diffs)

trunk/ippTasks/detrend.norm.pro

@@ -2 +2 @@
 ## this file contains the tasks for running the detrend normalization stage
 
-## these tasks use the queue DetrendNormalize
-## the DetrendNormalize queue contains:
+## these tasks use the queue DetrendNorm
+## the DetrendNorm queue contains:
 ## STATE DET_ID ITER
 
-queueinit DetrendNormalize
+queueinit DetrendNorm
 queueinit DetrendNormStat
-queueinit DetrendNormalizeExposures
+queueinit DetrendNormExp
 if ($?network == 0)
   $network = 1
@@ -14 +14 @@
 if ($?parallel == 0)
   $parallel = 0
+end
+
+$DetrendNormFail = 0
+$DetrendNormStatFail = 0
+$DetrendNormExpFail = 0
+
+if ($?VERBOSE == 0)
+  echo "VERBOSE not defined: load pantasks.pro first"
+  break
+end
+if ($?LOGDIR == 0)
+  echo "LOGDIR not defined: load pantasks.pro first"
+  break
+end
+$LOGSUBDIR = $LOGDIR/detrend
+exec mkdir -p $LOGSUBDIR
+
+macro detnorm.status
+  queueprint DetrendNorm
+  queueprint DetrendNormStat
+  queueprint DetrendNormExp
+  echo "DetrendNormFail: $DetrendNormFail"
+  echo "DetrendNormStatFail: $DetrendNormStatFail"
+  echo "DetrendNormExpFail: $DetrendNormExpFail"
 end
 
@@ -20 +44 @@
 # compare the new list with the ones already selected
 task           dettool.normstat.load
-  command      dettool -tonormalizedstat -simple
   host         local
 
@@ -27 +50 @@
   periods      -timeout 30
 
-  stdout dettool.normstat.log
-  stderr dettool.normstat.log
+  stdout $LOGSUBDIR/dettool.normstat.log
+  stderr $LOGSUBDIR/dettool.normstat.log
+
+  task.exec
+    command      dettool -tonormalizedstat -simple -limit {$DetrendNormStatFail + 20}
+  end
 
   # success
@@ -41 +68 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_ID   = $word:0
@@ -56 +86 @@
   # default exit status
   task.exit    default
-    echo       "detrend.norm: failure"
-  end
-
-  # operation times out?
-  task.exit    timeout
-    echo       "detrend.norm: timeout"
+    if ($VERBOSE) 
+      echo       "detrend.norm: failure"
+    end
+  end
+
+  # operation times out?
+  task.exit    timeout
+    if ($VERBOSE) 
+      echo       "detrend.norm: timeout"
+    end
   end
 end
@@ -73 +107 @@
   task.exec
     queuesize DetrendNormStat -var N
-   
     if ($N == 0) break
     if ($network == 0) break
@@ -95 +128 @@
 
     # XXX add $WORKDIR/$LOG_DIR
-    stdout $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
-    stderr $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
+    stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
+    stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
 
     # specify choice of remote host:
@@ -107 +140 @@
     # create example job options as a demonstration
     options "$line"
-    echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE
+    if ($VERBOSE > 1)
+      echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE
+    end
     command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE
   end
@@ -118 +153 @@
   # default exit status
   task.exit    default
+    if ($VERBOSE)
+      echo "failure detrend_norm_calc.pl $options:0"
+    end
     queuepush DetrendNormStat -replace -key 1:3 "FAIL $options:0"
-  end
-
-  # operation times out?
-  task.exit    timeout
+    $DetrendNormStatFail ++
+  end
+
+  # operation times out?
+  task.exit    timeout
+    if ($VERBOSE)
+      echo "failure detrend_norm_calc.pl $options:0"
+    end
     queuepush DetrendNormStat -replace -key 1:3 "TIMEOUT $options:0"
+   $DetrendNormStatFail ++
   end
 end
 
 # select images ready for copy 
-# new entries are added to queue DetrendNormalize
+# new entries are added to queue DetrendNorm
 # compare the new list with the ones already selected
 task           dettool.norm.load
-  command      dettool -tonormalize -simple
   host         local
 
@@ -138 +180 @@
   periods      -timeout 30
 
-  stdout dettool.norm.log
-  stderr dettool.norm.log
+  stdout $LOGSUBDIR/dettool.norm.log
+  stderr $LOGSUBDIR/dettool.norm.log
+
+  task.exec
+    command      dettool -tonormalize -simple -limit {$DetrendNormFail + 20}
+  end
 
   # success
@@ -152 +198 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_TYPE = $word:0
@@ -160 +209 @@
       $CLASS_ID = $word:5
       $NORM     = $word:6
-      queuepush DetrendNormalize -uniq -key 4:5 "NEW $DET_TYPE $CAMERA $URI $DET_ID $ITER $CLASS_ID $NORM"
+      queuepush DetrendNorm -uniq -key 4:5 "NEW $DET_TYPE $CAMERA $URI $DET_ID $ITER $CLASS_ID $NORM"
     end
 
     # delete existing entries which are DONE
-    queuedrop DetrendNormalize -key 0 DONE
+    queuedrop DetrendNorm -key 0 DONE
   end
 
@@ -170 +219 @@
   # default exit status
   task.exit    default
-    echo       "detrend.norm: failure"
-  end
-
-  # operation times out?
-  task.exit    timeout
-    echo       "detrend.norm: timeout"
+    if ($VERBOSE) 
+      echo       "detrend.norm: failure"
+    end
+  end
+
+  # operation times out?
+  task.exit    timeout
+    if ($VERBOSE) 
+      echo       "detrend.norm: timeout"
+    end
   end
 end
@@ -186 +239 @@
 
   task.exec
-    queuesize DetrendNormalize -var N
-   
+    queuesize DetrendNorm -var N
     if ($N == 0) break
     if ($network == 0) break
     
-    # look for new images on the internal DetrendNormalize queue
+    # look for new images on the internal DetrendNorm queue
     # caution with these 'if' statements: syntax errors 
     # will make the task fail without given a good status
-    queuepop DetrendNormalize -var line -key 0 NEW
+    queuepop DetrendNorm -var line -key 0 NEW
     if ("$line" == "NULL") break
 
     strpop line state
-    queuepush DetrendNormalize -replace -key 4:5 "RUN $line"
-
-    # the DetrendNormalize queue contains:
+    queuepush DetrendNorm -replace -key 4:5 "RUN $line"
+
+    # the DetrendNorm queue contains:
     # STATE DET_ID ITER
     list word -split $line
@@ -211 +263 @@
     $NORM     = $word:6
 
-    $LOG_DIR = `dirname $URI`
-
-    stdout $WORKDIR/$LOG_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
-    stderr $WORKDIR/$LOG_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
+    stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
+    stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
 
     # specify choice of remote host:
@@ -225 +275 @@
     # create example job options as a demonstration
     options "$line"
-    echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE
+    if ($VERBOSE > 1)
+      echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE
+    end
     command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE
   end
@@ -231 +283 @@
   # success
   task.exit 0
-    queuepush DetrendNormalize -replace -key 4:5 "DONE $options:0"
-  end
-
-  # default exit status
-  task.exit    default
-    queuepush DetrendNormalize -replace -key 4:5 "FAIL $options:0"
-  end
-
-  # operation times out?
-  task.exit    timeout
-    queuepush DetrendNormalize -replace -key 4:5 "TIMEOUT $options:0"
+    queuepush DetrendNorm -replace -key 4:5 "DONE $options:0"
+  end
+
+  # default exit status
+  task.exit    default
+    if ($VERBOSE)
+      echo "failure detrend_norm_apply.pl $options:0"
+    end
+    queuepush DetrendNorm -replace -key 4:5 "FAIL $options:0"
+    $DetrendNormFail ++
+  end
+
+  # operation times out?
+  task.exit    timeout
+    if ($VERBOSE)
+      echo "failure detrend_norm_apply.pl $options:0"
+    end
+    queuepush DetrendNorm -replace -key 4:5 "TIMEOUT $options:0"
+    $DetrendNormFail ++
   end
 end
 
 # select images ready for copy 
-# new entries are added to queue DetrendNormalizeExposures
+# new entries are added to queue DetrendNormExp
 # compare the new list with the ones already selected
 task           dettool.normexp.load
-  command      dettool -tonormalizedexp -simple
   host         local
 
@@ -256 +315 @@
   periods      -timeout 30
 
-  stdout dettool.normexp.log
-  stderr dettool.normexp.log
+  stdout $LOGSUBDIR/dettool.normexp.log
+  stderr $LOGSUBDIR/dettool.normexp.log
+
+  task.exec
+    command      dettool -tonormalizedexp -simple -limit {$DetrendNormExpFail + 20}
+  end
 
   # success
@@ -270 +333 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_ID   = $word:0
@@ -278 +344 @@
       #$EXP_TYPE = $word:5
       #$IMFILES  = $word:6
-      queuepush DetrendNormalizeExposures -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $CAMERA"
+      queuepush DetrendNormalizeExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $CAMERA"
     end
 
     # delete existing entries which are DONE
-    queuedrop DetrendNormalizeExposures -key 0 DONE
+    queuedrop DetrendNormExp -key 0 DONE
   end
 
@@ -288 +354 @@
   # default exit status
   task.exit    default
-    echo       "detrend.norm: failure"
-  end
-
-  # operation times out?
-  task.exit    timeout
-    echo       "detrend.norm: timeout"
+    if ($VERBOSE) 
+      echo       "detrend.norm: failure"
+    end
+  end
+
+  # operation times out?
+  task.exit    timeout
+    if ($VERBOSE) 
+      echo       "detrend.norm: timeout"
+    end
   end
 end
@@ -304 +374 @@
 
   task.exec
-    queuesize DetrendNormalizeExposures -var N
+    queuesize DetrendNormExp -var N
    
     if ($N == 0) break
     if ($network == 0) break
     
-    # look for new images on the internal DetrendNormalizeExposures queue
+    # look for new images on the internal DetrendNormExp queue
     # caution with these 'if' statements: syntax errors 
     # will make the task fail without given a good status
-    queuepop DetrendNormalizeExposures -var line -key 0 NEW
+    queuepop DetrendNormExp -var line -key 0 NEW
     if ("$line" == "NULL") break
 
     strpop line state
-    queuepush DetrendNormalizeExposures -replace -key 1:2 "RUN $line"
-
-    # the DetrendNormalizeExposures queue contains:
+    queuepush DetrendNormExp -replace -key 1:2 "RUN $line"
+
+    # the DetrendNormExp queue contains:
     # STATE DET_ID ITER
     list word -split $line
@@ -330 +400 @@
 
     # XXX add $WORKDIR/$LOG_DIR
-    stdout $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
-    stderr $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
+    stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
+    stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
 
     # specify choice of remote host:
@@ -342 +412 @@
     # create example job options as a demonstration
     options "$line"
-    echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE
+    if ($VERBOSE > 1)
+      echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE
+    end
     command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE
   end
@@ -348 +420 @@
   # success
   task.exit 0
-    queuepush DetrendNormalizeExposures -replace -key 1:2 "DONE $options:0"
-  end
-
-  # default exit status
-  task.exit    default
-    queuepush DetrendNormalizeExposures -replace -key 1:2 "FAIL $options:0"
-  end
-
-  # operation times out?
-  task.exit    timeout
-    queuepush DetrendNormalizeExposures -replace -key 1:2 "TIMEOUT $options:0"
-  end
-end
+    queuepush DetrendNormExp -replace -key 1:2 "DONE $options:0"
+  end
+
+  # default exit status
+  task.exit    default
+    if ($VERBOSE)
+      echo "failure detrend_norm_exp.pl $options:0"
+    end
+    queuepush DetrendNormExp -replace -key 1:2 "FAIL $options:0"
+   $DetrendNormExpFail ++
+  end
+
+  # operation times out?
+  task.exit    timeout
+    if ($VERBOSE)
+      echo "failure detrend_norm_exp.pl $options:0"
+    end
+    queuepush DetrendNormExp -replace -key 1:2 "TIMEOUT $options:0"
+   $DetrendNormExpFail ++
+  end
+end

trunk/ippTasks/detrend.process.pro

@@ -22 +22 @@
 end
 
+$DetrendProcessImfileFail = 0
+$DetrendProcessExpFail = 0
+
+if ($?VERBOSE == 0)
+  echo "VERBOSE not defined: load pantasks.pro first"
+  break
+end
+if ($?LOGDIR == 0)
+  echo "LOGDIR not defined: load pantasks.pro first"
+  break
+end
+$LOGSUBDIR = $LOGDIR/detrend
+exec mkdir -p $LOGSUBDIR
+
+macro detproc.status
+  queueprint DetrendProcessImfiles
+  queueprint DetrendProcessExposures
+  echo "DetrendProcessImfileFail: $DetrendProcessImfileFail"
+  echo "DetrendProcessExpFail: $DetrendProcessExpFail"
+end
+
 # select images ready for copy 
 # new entries are added to queue DetrendProcessImfiles
 # compare the new list with the ones already selected
 task           dettool.process.load
-  command      dettool -toprocessedimfile -simple
   host         local
 
@@ -33 +53 @@
   periods      -timeout 30
 
-  stdout dettool.process.log
-  stderr dettool.process.log
+  stdout $LOGSUBDIR/detproc.imfile.log
+  stderr $LOGSUBDIR/detproc.imfile.log
+
+  task.exec
+    command dettool -toprocessedimfile -simple -limit {$DetrendProcessImfileFail + 20}
+  end
 
   # success
@@ -47 +71 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_ID   = $word:0
@@ -64 +91 @@
   # error
   task.exit    default
-    echo       "detrend.process: failure"
+    if ($VERBOSE) 
+      echo       "detrend.process: failure"
+    end
   end
 
   # operation times out?
   task.exit    timeout
-    echo       "detrend.process: timeout"
+    if ($VERBOSE) 
+      echo       "detrend.process: timeout"
+    end
   end
 end
@@ -105 +136 @@
     $CAMERA   = $word:6
 
-    $LOG_DIR = `dirname $URI`
-    stdout $WORKDIR/$LOG_DIR/$EXP_TAG.detproc.$DET_ID.$CLASS_ID.log
-    stderr $WORKDIR/$LOG_DIR/$EXP_TAG.detproc.$DET_ID.$CLASS_ID.log
+    $word = `basename $EXP_TAG | tr '.' ' '`
+    list word -split $word
+    $base = $word:0
+    stdout $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.$CLASS_ID.log
+    stderr $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.$CLASS_ID.log
+    exec mkdir -p $LOGSUBDIR/$base
 
     # specify choice of remote host:
@@ -118 +152 @@
     # create example job options as a demonstration
     options "$line"
-    echo command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA
+    if ($VERBOSE > 1)
+      echo command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA
+    end
     command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA
   end
@@ -130 +166 @@
   # default exit status
   task.exit    default
-    echo "failure detrend_process_imfile.pl $options:0"
+    if ($VERBOSE)
+      echo "failure detrend_process_imfile.pl $options:0"
+    end
     queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0"
+    $DetrendProcessImfileFail ++
   end
 
   # operation times out?
   task.exit    timeout
-    echo "timeout detrend_process_imfile.pl $options:0"
-    queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0"
-  end
+    if ($VERBOSE)
+      echo "timeout detrend_process_imfile.pl $options:0"
+    end
+    queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0" 
+    $DetrendProcessImfileFail ++
+ end
 end
 
@@ -145 +187 @@
 # compare the new list with the ones already selected
 task           dettool.processexp.load
-  command      dettool -toprocessedexp -simple
   host         local
 
@@ -152 +193 @@
   periods      -timeout 30
 
-  stdout dettool.processexp.log
-  stderr dettool.processexp.log
+  stdout $LOGSUBDIR/detproc.exp.log
+  stderr $LOGSUBDIR/detproc.exp.log
+
+  task.exec
+    command dettool -toprocessedexp -simple -limit {$DetrendProcessExpFail + 20}
+  end
 
   # success
@@ -166 +211 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_ID   = $word:0
@@ -181 +229 @@
   # error
   task.exit    default
-    echo       "detrend.process: failure"
+    if ($VERBOSE)
+      echo       "detrend.process: failure"
+    end
   end
 
   # operation times out?
   task.exit    timeout
-    echo       "detrend.process: timeout"
+    if ($VERBOSE)
+      echo       "detrend.process: timeout"
+    end
   end
 end
@@ -210 +262 @@
     queuepush DetrendProcessExposures -replace -key 1:2:4 "RUN $line"
 
-
     list word -split $line
     $DET_ID   = $word:0
@@ -218 +269 @@
     $CAMERA   = $word:4
 
-    # XXX need the classID in dettool -toprocessedexp -simple output
-    $CLASS_ID = isp
-    stdout $EXP_TAG.detproc.$DET_ID.log
-    stderr $EXP_TAG.detproc.$DET_ID.log
+    # XXX this is still a lame rule
+    $word = `basename $EXP_TAG | tr '.' ' '`
+    list word -split $word
+    $base = $word:0
+
+    stdout $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log
+    stderr $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log
+    exec mkdir -p $LOGSUBDIR/$base
 
     # specify choice of remote host:
@@ -232 +287 @@
     # create example job options as a demonstration
     options "$line"
-    echo command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA
+    if ($VERBOSE > 1)
+      echo command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA
+    end
     command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA
   end
@@ -243 +300 @@
   # default exit status
   task.exit    default
+    if ($VERBOSE)
+      echo "failure for detrend_process_exp.pl: $options:0"
+    end
     queuepush DetrendProcessExposures -replace -key 1:2:4 "FAIL $options:0"
+    $DetrendProcessExpFail ++
   end
 
   # operation times out?
   task.exit    timeout
+    if ($VERBOSE)
+      echo "failure for detrend_process_exp.pl: $options:0"
+    end
     queuepush DetrendProcessExposures -replace -key 1:2:4 "TIMEOUT $options:0"
-  end
-end
+    $DetrendProcessExpFail ++
+  end
+end

trunk/ippTasks/detrend.reject.pro

@@ -13 +13 @@
 ## STATE DET_ID ITER TYPE
 
-queueinit DetrendRejectExposure
+queueinit DetrendRejectExp
 if ($?network == 0)
   $network = 1
@@ -21 +21 @@
 end
 
+$DetrendRejectExpFail = 0
+
+if ($?VERBOSE == 0)
+  echo "VERBOSE not defined: load pantasks.pro first"
+  break
+end
+if ($?LOGDIR == 0)
+  echo "LOGDIR not defined: load pantasks.pro first"
+  break
+end
+$LOGSUBDIR = $LOGDIR/detrend
+exec mkdir -p $LOGSUBDIR
+
+macro detreject.status
+  queueprint DetrendRejectExp
+  echo "DetrendRejectExpFail: $DetrendRejectExpFail"
+end
+
 # select images ready for copy 
-# new entries are added to queue DetrendRejectExposure
+# new entries are added to queue DetrendRejectExp
 # compare the new list with the ones already selected
 task           dettool.reject.load
-  command      dettool -residdetrun -simple
   host         local
 
@@ -32 +49 @@
   periods      -timeout 30
 
-  stdout dettool.reject.log
-  stderr dettool.reject.log
+  stdout $LOGSUBDIR/dettool.reject.log
+  stderr $LOGSUBDIR/dettool.reject.log
+
+  task.exec
+    command dettool -residdetrun -simple -limit {$DetrendRejectExpFail + 20}
+  end
 
   # success
@@ -44 +65 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_ID   = $word:0
@@ -50 +74 @@
       $MODE     = $word:3
       $CAMERA   = $word:4
-      queuepush DetrendRejectExposure -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $MODE $CAMERA"
+      queuepush DetrendRejectExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $MODE $CAMERA"
     end
 
     # delete existing entries which are DONE
-    queuedrop DetrendRejectExposure -key 0 DONE
+    queuedrop DetrendRejectExp -key 0 DONE
   end
 
   # locked list
   task.exit    default
-    echo       "detrend.reject: failure"
+    if ($VERBOSE) 
+      echo       "detrend.reject: failure"
+    end
   end
 
   # operation times out?
   task.exit    timeout
-    echo       "detrend.reject: timeout"
+    if ($VERBOSE) 
+      echo       "detrend.reject: timeout"
+    end
   end
 end
@@ -75 +103 @@
 
   task.exec
-    queuesize DetrendRejectExposure -var N
+    queuesize DetrendRejectExp -var N
     if ($N == 0) break
     if ($network == 0) break
     
-    # look for new images on the internal DetrendRejectExposure queue
+    # look for new images on the internal DetrendRejectExp queue
     # caution with these 'if' statements: syntax errors 
     # will make the task fail without given a good status
-    queuepop DetrendRejectExposure -var line -key 0 NEW
+    queuepop DetrendRejectExp -var line -key 0 NEW
     if ("$line" == "NULL") break
 
     strpop line state
-    queuepush DetrendRejectExposure -replace -key 1:2 "RUN $line"
+    queuepush DetrendRejectExp -replace -key 1:2 "RUN $line"
 
-    ## the DetrendRejectExposure queue contains:
+    ## the DetrendRejectExp queue contains:
     ## STATE DET_ID ITER TYPE
     ## NOTE: the queue has the STATE prepended 
@@ -98 +126 @@
 
     # I'd like to add CAMERA to the log file...
-    stdout $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
-    stderr $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
+    stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
+    stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
 
     # specify choice of remote host:
@@ -110 +138 @@
     # create example job options as a demonstration
     options "$line"
-    echo command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE --camera $CAMERA
+    if ($VERBOSE > 1)
+      echo command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE --camera $CAMERA
+    end
     command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE --camera $CAMERA
  end
@@ -116 +146 @@
   # success
   task.exit 0
-    queuepush DetrendRejectExposure -replace -key 1:2 "DONE $options:0"
+    queuepush DetrendRejectExp -replace -key 1:2 "DONE $options:0"
   end
 
   # default exit status
   task.exit    default
-    queuepush DetrendRejectExposure -replace -key 1:2 "FAIL $options:0"
+    if ($VERBOSE)
+      echo "failure detrend_reject_exp.pl $options:0"
+    end
+    queuepush DetrendRejectExp -replace -key 1:2 "FAIL $options:0"
+    $DetrendRejectExpFail ++
   end
 
   # operation times out?
   task.exit    timeout
-    queuepush DetrendRejectExposure -replace -key 1:2 "TIMEOUT $options:0"
+    if ($VERBOSE)
+      echo "failure detrend_reject_exp.pl $options:0"
+    end
+    queuepush DetrendRejectExp -replace -key 1:2 "TIMEOUT $options:0"
+   $DetrendRejectExpFail ++
   end
 end

trunk/ippTasks/detrend.resid.pro

@@ -23 +23 @@
 end
 
+$DetrendResidImfileFail = 0
+$DetrendResidExpFail = 0
+
+if ($?VERBOSE == 0)
+  echo "VERBOSE not defined: load pantasks.pro first"
+  break
+end
+if ($?LOGDIR == 0)
+  echo "LOGDIR not defined: load pantasks.pro first"
+  break
+end
+$LOGSUBDIR = $LOGDIR/detrend
+exec mkdir -p $LOGSUBDIR
+
+macro detresid.status
+  queueprint DetrendResidImfiles
+  queueprint DetrendResidExposures
+  echo "DetrendResidImfileFail: $DetrendResidImfileFail"
+  echo "DetrendResidExpFail: $DetrendResidExpFail"
+end
+
 # select images ready for copy 
 # new entries are added to queue DetrendResidImfiles
 # compare the new list with the ones already selected
 task           dettool.resid.load
-  command      dettool -toresidimfile -simple
   host         local
 
@@ -34 +54 @@
   periods      -timeout 30
 
-  stdout dettool.resid.log
-  stderr dettool.resid.log
+  stdout $LOGSUBDIR/detresid.log
+  stderr $LOGSUBDIR/detresid.log
+
+  task.exec
+    command dettool -toresidimfile -simple -limit {$DetrendResidImfileFail + 20}
+  end
 
   # success
@@ -48 +72 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_ID   = $word:0
@@ -70 +97 @@
   # default exit status
   task.exit    default
-    echo       "detrend.resid: failure"
+    if ($VERBOSE) 
+      echo       "detrend.resid: failure"
+    end
   end
 
   # operation times out?
   task.exit    timeout
-    echo       "detrend.resid: timeout"
+    if ($VERBOSE) 
+      echo       "detrend.resid: timeout"
+    end
   end
 end
@@ -113 +144 @@
     $CAMERA   = $word:8
 
-    $DET_DIR = `dirname $URI_IN`
-    $LOG_DIR = $WORKDIR/$DET_DIR
-
-    stdout $LOG_DIR/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log
-    stderr $LOG_DIR/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log
+    $word = `basename $EXP_TAG | tr '.' ' '`
+    list word -split $word
+    $base = $word:0
+    stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log
+    stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log
 
     # specify choice of remote host:
@@ -130 +161 @@
     # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG ???)
     # need to parse the subdir path for the detrend URL
-    $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
-    echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URL --input_uri $URI_IN --camera $CAMERA --mode $MODE
-    command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URL --input_uri $URI_IN --camera $CAMERA --mode $MODE
+    # $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
+    echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE
+    command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE
   end
 
@@ -142 +173 @@
   # default exit status
   task.exit    default
+    if ($VERBOSE)
+      echo "failure detrend_resid.pl $options:0"
+    end
     queuepush DetrendResidImfiles -replace -key 1:2:5:6 "FAIL $options:0"
+    $DetrendResidImfileFail ++
   end
 
   # operation times out?
   task.exit    timeout
+    if ($VERBOSE)
+      echo "failure detrend_resid.pl $options:0"
+    end
     queuepush DetrendResidImfiles -replace -key 1:2:6:6 "TIMEOUT $options:0"
+    $DetrendResidImfileFail ++
   end
 end
@@ -155 +194 @@
 # compare the new list with the ones already selected
 task           dettool.residexp.load
-  command      dettool -toresidexp -simple
   host         local
 
@@ -162 +200 @@
   periods      -timeout 30
 
-  stdout dettool.residexp.log
-  stderr dettool.residexp.log
+  stdout $LOGSUBDIR/detresid.exp.log
+  stderr $LOGSUBDIR/detresid.exp.log
+
+  task.exec
+  command      dettool -toresidexp -simple -limit {$DetrendResidExpFail + 20}
+  end
 
   # success
@@ -176 +218 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_ID   = $word:0
@@ -194 +239 @@
   # default exit status
   task.exit    default
-    echo       "detrend.residexp: failure"
+    if ($VERBOSE)
+      echo       "detrend.residexp: failure"
+    end
   end
 
   # operation times out?
   task.exit    timeout
-    echo       "detrend.residexp: timeout"
+    if ($VERBOSE)
+      echo       "detrend.residexp: timeout"
+    end
   end
 end
@@ -235 +284 @@
     $CAMERA   = $word:6
 
-    # XXX add $WORKDIR/$LOG_DIR
-    stdout $EXP_TAG.detresid.$DET_ID.$ITER.log
-    stderr $EXP_TAG.detresid.$DET_ID.$ITER.log
+    # XXX this is still a lame rule
+    $word = `basename $EXP_TAG | tr '.' ' '`
+    list word -split $word
+    $base = $word:0
+
+    stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log
+    stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log
+    exec mkdir -p $LOGSUBDIR/$base
 
     # specify choice of remote host:
@@ -248 +302 @@
     # create example job options as a demonstration
     options "$line"
-    echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA
+    if ($VERBOSE > 1)
+      echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA
+    end
     command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA
   end
@@ -259 +315 @@
   # default exit status
   task.exit    default
+    if ($VERBOSE)
+      echo "failure for detrend_reject_imfile.pl: $options:0"
+    end
     queuepush DetrendResidExposures -replace -key 1:2:5 "FAIL $options:0"
+    $DetrendResidExpFail ++
   end
 
   # operation times out?
   task.exit    timeout
+    if ($VERBOSE)
+      echo "failure for detrend_reject_imfile.pl: $options:0"
+    end
     queuepush DetrendResidExposures -replace -key 1:2:5 "TIMEOUT $options:0"
-  end
-end
+    $DetrendResidExpFail ++
+  end
+end

trunk/ippTasks/detrend.stack.pro

@@ -9 +9 @@
 # detrend_stack.pl 1 0 mc bias
 
-## these tasks use the queue DetrendClassIDtoStack
-
-## these tasks use the queue DetrendImfilesToProcess
-## the DetrendClassIDtoStack queue contains:
+## these tasks use the queue DetrendStackClass
+## the DetrendStackClass queue contains:
 ## STATE DET_ID ITER TYPE CLASS_ID CAMERA
 
@@ -23 +21 @@
 end
 
+$DetrendStackClassFail = 0
+
+if ($?VERBOSE == 0)
+  echo "VERBOSE not defined: load pantasks.pro first"
+  break
+end
+if ($?LOGDIR == 0)
+  echo "LOGDIR not defined: load pantasks.pro first"
+  break
+end
+$LOGSUBDIR = $LOGDIR/detrend
+exec mkdir -p $LOGSUBDIR
+
+macro detstack.status
+  queueprint DetrendStackClass
+  echo "DetrendStackClassFail: $DetrendStackClassFail"
+end
+
 # select images ready for copy 
 # new entries are added to queue DetrendStackClass
 # compare the new list with the ones already selected
 task           dettool.stack.load
-  command      dettool -tostacked -simple
   host         local
 
@@ -34 +49 @@
   periods      -timeout 30
 
-  stdout dettool.stack.log
-  stderr dettool.stack.log
+  stdout $LOGSUBDIR/detstack.log
+  stderr $LOGSUBDIR/detstack.log
+
+  task.exec
+    command dettool -tostacked -simple -limit {$DetrendStackClassFail + 20}
+  end
 
   # success
@@ -46 +65 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_ID   = $word:0
@@ -61 +83 @@
   # locked list
   task.exit    default
-    echo       "detrend.stack: failure"
+    if ($VERBOSE) 
+      echo       "detrend.stack: failure"
+    end
   end
 
   # operation times out?
   task.exit    timeout
-    echo       "detrend.stack: timeout"
+    if ($VERBOSE) 
+      echo       "detrend.stack: timeout"
+    end
   end
 end
@@ -78 +104 @@
   task.exec
     queuesize DetrendStackClass -var N
-   
     if ($N == 0) break
     if ($network == 0) break
@@ -103 +128 @@
 
     # XXX add $WORKDIR/$LOG_DIR
-    stdout $CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log
-    stderr $CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log
+    stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log
+    stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log
 
     # specify choice of remote host:
@@ -115 +140 @@
     # create example job options as a demonstration
     options "$line"
-    # detrend_stack.pl --det_id 1 --iteration 0 --class_id isp --det_type bias --camera ISP || exit 1
-    echo command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA
+    if ($VERBOSE > 1)
+      echo command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA
+    end
     command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA
   end
@@ -127 +153 @@
   # default exit status
   task.exit    default
-    queuepush DetrendStackClass -replace -key 1:2:4 "FAIL $options:0"
+    if ($VERBOSE)
+      echo "failure detrend_stack.pl $options:0"
+    end
+   queuepush DetrendStackClass -replace -key 1:2:4 "FAIL $options:0"
+   $DetrendStackClassFail ++
   end
 
   # operation times out?
   task.exit    timeout
+    if ($VERBOSE)
+      echo "failure detrend_stack.pl $options:0"
+    end
     queuepush DetrendStackClass -replace -key 1:2:4 "TIMEOUT $options:0"
+    $DetrendStackClassFail ++
   end
 end

trunk/ippTasks/phase0.pro

@@ -40 +40 @@
 # compare the new list with the ones already selected
 task           phase0.imfile.load
-  command      p0tool -pendingimfile -simple
   host         local
 
@@ -83 +82 @@
       echo "phase0 pending imfiles: failure"
     end
-    $Phase0ImfileFail ++
   end
 
@@ -93 +91 @@
   end
 end
+
+
 
 # run the phase0imfile script on pending images
@@ -166 +166 @@
     end
     queuepush Phase0Imfiles -replace -key 1 "TIMEOUT $options:0"
-  end
-end
+    $Phase0ImfileFail ++
+  end
+end
+
+
 
 ## these tasks use the queue Phase0Exposures
@@ -199 +202 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $EXP_TAG   = $word:0
@@ -228 +234 @@
   end
 end
+
+
 
 # run the phase0imfile script on pending images
@@ -285 +293 @@
   # default exit status
   task.exit    default
+    if ($VERBOSE)
+      echo "failure for phase0exp.pl: $options:0"
+    end
     queuepush Phase0Exposures -replace -key 1 "FAIL $options:0"
     $Phase0ExpFail
@@ -291 +302 @@
   # operation times out?
   task.exit    timeout
+    if ($VERBOSE)
+      echo "failure for phase0exp.pl: $options:0"
+    end
     queuepush Phase0Exposures -replace -key 1 "TIMEOUT $options:0"
+    $Phase0ExpFail
   end
 end