Changeset 10692 for trunk/ippTasks/detrend.resid.pro

Timestamp:

Dec 12, 2006, 11:58:21 PM (20 years ago)

Author:

eugene

Message:

added global logdirs, VERBOSE output, failure tracking

File:

• trunk/ippTasks/detrend.resid.pro (modified) (14 diffs)

trunk/ippTasks/detrend.resid.pro

@@ -23 +23 @@
 end
 
+$DetrendResidImfileFail = 0
+$DetrendResidExpFail = 0
+
+if ($?VERBOSE == 0)
+  echo "VERBOSE not defined: load pantasks.pro first"
+  break
+end
+if ($?LOGDIR == 0)
+  echo "LOGDIR not defined: load pantasks.pro first"
+  break
+end
+$LOGSUBDIR = $LOGDIR/detrend
+exec mkdir -p $LOGSUBDIR
+
+macro detresid.status
+  queueprint DetrendResidImfiles
+  queueprint DetrendResidExposures
+  echo "DetrendResidImfileFail: $DetrendResidImfileFail"
+  echo "DetrendResidExpFail: $DetrendResidExpFail"
+end
+
 # select images ready for copy 
 # new entries are added to queue DetrendResidImfiles
 # compare the new list with the ones already selected
 task           dettool.resid.load
-  command      dettool -toresidimfile -simple
   host         local
 
@@ -34 +54 @@
   periods      -timeout 30
 
-  stdout dettool.resid.log
-  stderr dettool.resid.log
+  stdout $LOGSUBDIR/detresid.log
+  stderr $LOGSUBDIR/detresid.log
+
+  task.exec
+    command dettool -toresidimfile -simple -limit {$DetrendResidImfileFail + 20}
+  end
 
   # success
@@ -48 +72 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_ID   = $word:0
@@ -70 +97 @@
   # default exit status
   task.exit    default
-    echo       "detrend.resid: failure"
+    if ($VERBOSE) 
+      echo       "detrend.resid: failure"
+    end
   end
 
   # operation times out?
   task.exit    timeout
-    echo       "detrend.resid: timeout"
+    if ($VERBOSE) 
+      echo       "detrend.resid: timeout"
+    end
   end
 end
@@ -113 +144 @@
     $CAMERA   = $word:8
 
-    $DET_DIR = `dirname $URI_IN`
-    $LOG_DIR = $WORKDIR/$DET_DIR
-
-    stdout $LOG_DIR/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log
-    stderr $LOG_DIR/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log
+    $word = `basename $EXP_TAG | tr '.' ' '`
+    list word -split $word
+    $base = $word:0
+    stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log
+    stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log
 
     # specify choice of remote host:
@@ -130 +161 @@
     # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG ???)
     # need to parse the subdir path for the detrend URL
-    $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
-    echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URL --input_uri $URI_IN --camera $CAMERA --mode $MODE
-    command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URL --input_uri $URI_IN --camera $CAMERA --mode $MODE
+    # $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
+    echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE
+    command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE
   end
 
@@ -142 +173 @@
   # default exit status
   task.exit    default
+    if ($VERBOSE)
+      echo "failure detrend_resid.pl $options:0"
+    end
     queuepush DetrendResidImfiles -replace -key 1:2:5:6 "FAIL $options:0"
+    $DetrendResidImfileFail ++
   end
 
   # operation times out?
   task.exit    timeout
+    if ($VERBOSE)
+      echo "failure detrend_resid.pl $options:0"
+    end
     queuepush DetrendResidImfiles -replace -key 1:2:6:6 "TIMEOUT $options:0"
+    $DetrendResidImfileFail ++
   end
 end
@@ -155 +194 @@
 # compare the new list with the ones already selected
 task           dettool.residexp.load
-  command      dettool -toresidexp -simple
   host         local
 
@@ -162 +200 @@
   periods      -timeout 30
 
-  stdout dettool.residexp.log
-  stderr dettool.residexp.log
+  stdout $LOGSUBDIR/detresid.exp.log
+  stderr $LOGSUBDIR/detresid.exp.log
+
+  task.exec
+  command      dettool -toresidexp -simple -limit {$DetrendResidExpFail + 20}
+  end
 
   # success
@@ -176 +218 @@
     for i 0 $Nqueue
       queuepop stdout -var line
+      if ($VERBOSE > 2)
+        echo $line
+      end
       list word -split $line
       $DET_ID   = $word:0
@@ -194 +239 @@
   # default exit status
   task.exit    default
-    echo       "detrend.residexp: failure"
+    if ($VERBOSE)
+      echo       "detrend.residexp: failure"
+    end
   end
 
   # operation times out?
   task.exit    timeout
-    echo       "detrend.residexp: timeout"
+    if ($VERBOSE)
+      echo       "detrend.residexp: timeout"
+    end
   end
 end
@@ -235 +284 @@
     $CAMERA   = $word:6
 
-    # XXX add $WORKDIR/$LOG_DIR
-    stdout $EXP_TAG.detresid.$DET_ID.$ITER.log
-    stderr $EXP_TAG.detresid.$DET_ID.$ITER.log
+    # XXX this is still a lame rule
+    $word = `basename $EXP_TAG | tr '.' ' '`
+    list word -split $word
+    $base = $word:0
+
+    stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log
+    stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log
+    exec mkdir -p $LOGSUBDIR/$base
 
     # specify choice of remote host:
@@ -248 +302 @@
     # create example job options as a demonstration
     options "$line"
-    echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA
+    if ($VERBOSE > 1)
+      echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA
+    end
     command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA
   end
@@ -259 +315 @@
   # default exit status
   task.exit    default
+    if ($VERBOSE)
+      echo "failure for detrend_reject_imfile.pl: $options:0"
+    end
     queuepush DetrendResidExposures -replace -key 1:2:5 "FAIL $options:0"
+    $DetrendResidExpFail ++
   end
 
   # operation times out?
   task.exit    timeout
+    if ($VERBOSE)
+      echo "failure for detrend_reject_imfile.pl: $options:0"
+    end
     queuepush DetrendResidExposures -replace -key 1:2:5 "TIMEOUT $options:0"
-  end
-end
+    $DetrendResidExpFail ++
+  end
+end