Changeset 10692 for trunk/ippTasks/detrend.process.pro

Timestamp:

Dec 12, 2006, 11:58:21 PM (20 years ago)

Author:

eugene

Message:

added global logdirs, VERBOSE output, failure tracking

File:

• trunk/ippTasks/detrend.process.pro (modified) (15 diffs)

trunk/ippTasks/detrend.process.pro

@@ -22 +22 @@
 end
 
+$DetrendProcessImfileFail = 0
+$DetrendProcessExpFail = 0
+
+if ($?VERBOSE == 0)
+  echo "VERBOSE not defined: load pantasks.pro first"
+  break
+end
+if ($?LOGDIR == 0)
+  echo "LOGDIR not defined: load pantasks.pro first"
+  break
+end
+$LOGSUBDIR = $LOGDIR/detrend
+exec mkdir -p $LOGSUBDIR
+
+macro detproc.status
+  queueprint DetrendProcessImfiles
+  queueprint DetrendProcessExposures
+  echo "DetrendProcessImfileFail: $DetrendProcessImfileFail"
+  echo "DetrendProcessExpFail: $DetrendProcessExpFail"
+end
+
 # select images ready for copy 
 # new entries are added to queue DetrendProcessImfiles
 # compare the new list with the ones already selected
 task           dettool.process.load
-  command      dettool -toprocessedimfile -simple
   host         local
 
@@ -33 +53 @@
   periods      -timeout 30
 
-  stdout dettool.process.log
-  stderr dettool.process.log
+  stdout $LOGSUBDIR/detproc.imfile.log
+  stderr $LOGSUBDIR/detproc.imfile.log
+
+  task.exec
+    command dettool -toprocessedimfile -simple -limit {$DetrendProcessImfileFail + 20}
+  end
 
   # success
@@ -47 +71 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_ID   = $word:0
@@ -64 +91 @@
   # error
   task.exit    default
-    echo       "detrend.process: failure"
+    if ($VERBOSE) 
+      echo       "detrend.process: failure"
+    end
   end
 
   # operation times out?
   task.exit    timeout
-    echo       "detrend.process: timeout"
+    if ($VERBOSE) 
+      echo       "detrend.process: timeout"
+    end
   end
 end
@@ -105 +136 @@
     $CAMERA   = $word:6
 
-    $LOG_DIR = `dirname $URI`
-    stdout $WORKDIR/$LOG_DIR/$EXP_TAG.detproc.$DET_ID.$CLASS_ID.log
-    stderr $WORKDIR/$LOG_DIR/$EXP_TAG.detproc.$DET_ID.$CLASS_ID.log
+    $word = `basename $EXP_TAG | tr '.' ' '`
+    list word -split $word
+    $base = $word:0
+    stdout $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.$CLASS_ID.log
+    stderr $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.$CLASS_ID.log
+    exec mkdir -p $LOGSUBDIR/$base
 
     # specify choice of remote host:
@@ -118 +152 @@
     # create example job options as a demonstration
     options "$line"
-    echo command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA
+    if ($VERBOSE > 1)
+      echo command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA
+    end
     command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA
   end
@@ -130 +166 @@
   # default exit status
   task.exit    default
-    echo "failure detrend_process_imfile.pl $options:0"
+    if ($VERBOSE)
+      echo "failure detrend_process_imfile.pl $options:0"
+    end
     queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0"
+    $DetrendProcessImfileFail ++
   end
 
   # operation times out?
   task.exit    timeout
-    echo "timeout detrend_process_imfile.pl $options:0"
-    queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0"
-  end
+    if ($VERBOSE)
+      echo "timeout detrend_process_imfile.pl $options:0"
+    end
+    queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0" 
+    $DetrendProcessImfileFail ++
+ end
 end
 
@@ -145 +187 @@
 # compare the new list with the ones already selected
 task           dettool.processexp.load
-  command      dettool -toprocessedexp -simple
   host         local
 
@@ -152 +193 @@
   periods      -timeout 30
 
-  stdout dettool.processexp.log
-  stderr dettool.processexp.log
+  stdout $LOGSUBDIR/detproc.exp.log
+  stderr $LOGSUBDIR/detproc.exp.log
+
+  task.exec
+    command dettool -toprocessedexp -simple -limit {$DetrendProcessExpFail + 20}
+  end
 
   # success
@@ -166 +211 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_ID   = $word:0
@@ -181 +229 @@
   # error
   task.exit    default
-    echo       "detrend.process: failure"
+    if ($VERBOSE)
+      echo       "detrend.process: failure"
+    end
   end
 
   # operation times out?
   task.exit    timeout
-    echo       "detrend.process: timeout"
+    if ($VERBOSE)
+      echo       "detrend.process: timeout"
+    end
   end
 end
@@ -210 +262 @@
     queuepush DetrendProcessExposures -replace -key 1:2:4 "RUN $line"
 
-
     list word -split $line
     $DET_ID   = $word:0
@@ -218 +269 @@
     $CAMERA   = $word:4
 
-    # XXX need the classID in dettool -toprocessedexp -simple output
-    $CLASS_ID = isp
-    stdout $EXP_TAG.detproc.$DET_ID.log
-    stderr $EXP_TAG.detproc.$DET_ID.log
+    # XXX this is still a lame rule
+    $word = `basename $EXP_TAG | tr '.' ' '`
+    list word -split $word
+    $base = $word:0
+
+    stdout $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log
+    stderr $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log
+    exec mkdir -p $LOGSUBDIR/$base
 
     # specify choice of remote host:
@@ -232 +287 @@
     # create example job options as a demonstration
     options "$line"
-    echo command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA
+    if ($VERBOSE > 1)
+      echo command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA
+    end
     command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA
   end
@@ -243 +300 @@
   # default exit status
   task.exit    default
+    if ($VERBOSE)
+      echo "failure for detrend_process_exp.pl: $options:0"
+    end
     queuepush DetrendProcessExposures -replace -key 1:2:4 "FAIL $options:0"
+    $DetrendProcessExpFail ++
   end
 
   # operation times out?
   task.exit    timeout
+    if ($VERBOSE)
+      echo "failure for detrend_process_exp.pl: $options:0"
+    end
     queuepush DetrendProcessExposures -replace -key 1:2:4 "TIMEOUT $options:0"
-  end
-end
+    $DetrendProcessExpFail ++
+  end
+end