Changeset 9081 for trunk/ippTasks/detrend.resid.pro

Timestamp:

Sep 30, 2006, 12:27:01 PM (20 years ago)

Author:

eugene

Message:

basic detrend stages work

File:

• trunk/ippTasks/detrend.resid.pro (modified) (6 diffs)

trunk/ippTasks/detrend.resid.pro

@@ -13 +13 @@
 ## the DetrendResidImfiles queue contains:
 ## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN
+
+queueinit DetrendResidImfiles
+queueinit DetrendResidExp
+if ($?network == 0)
+  $network = 1
+end
+if ($?parallel == 0)
+  $parallel = 0
+end
 
 # select images ready for copy 
@@ -23 +32 @@
   periods      -poll 1
   periods      -exec 5
-  periods      -timeout 30
+  periods      -timeout 5
+
+  stdout dettool.resid.log
+  stderr dettool.resid.log
 
   # success
@@ -38 +50 @@
       list word -split $line
       $ITER     = $word:0
-      $TYPE     = $word:1
+      $DET_TYPE = $word:1
       $DET_ID   = $word:2
       $EXP_KEY  = $word:3
       $CLASS_ID = $word:4
       $URI_IN   = $word:5
-      queuepush DetrendResidImfiles -uniq -key 1:3:4:5 "NEW $ITER $TYPE $DET_ID $EXP_KEY $CLASS_ID $URI_IN"
+      ## XXX need to include the detID and iter in the key (if we have two running at the same time...)
+      ## Also needed to be in the output file (or path)
+      queuepush DetrendResidImfiles -uniq -key 1:3:4:5 "NEW $ITER $DET_TYPE $DET_ID $EXP_KEY $CLASS_ID $URI_IN"
     end
 
@@ -69 +83 @@
 task           dettool.resid.process
   periods      -poll 0.5
-  periods      -exec 1.0
-  periods      -timeout 5
+  periods      -exec 5.0
+  periods      -timeout 15
 
   task.exec
@@ -90 +104 @@
     ## NOTE: the queue has the STATE prepended 
     list word -split $line
-    $ITER     = $word:1
-    $TYPE     = $word:2
-    $DET_ID   = $word:3
-    $EXP_KEY  = $word:4
-    $CLASS_ID = $word:5
-    $URI_IN   = $word:6
+    $ITER     = $word:0
+    $DET_TYPE = $word:1
+    $DET_ID   = $word:2
+    $EXP_KEY  = $word:3
+    $CLASS_ID = $word:4
+    $URI_IN   = $word:5
+
+    stdout $EXP_KEY.detresid.$DET_ID.$ITER.$CLASS_ID.log
+    stderr $EXP_KEY.detresid.$DET_ID.$ITER.$CLASS_ID.log
 
     # specify choice of remote host:
-    host anyhost
+    if ($parallel)
+      host anyhost
+    else
+      host local
+    end
 
     # create example job options as a demonstration
     options "$line"
     # detrend_create_resid.pl DET_ID ITER EXP_KEY CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_KEY ???)
-    $URL_DET = $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit
-    command detrend_create_resid.pl $DET_ID $ITER $EXP_KEY $CLASS_ID $TYPE $URI_DET $URL_IN $EXP_KEY
+    $URL_DET = isp.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
+    # $URL_DET = $CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
+    echo command detrend_create_resid.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --class_id $CLASS_ID --det_type $DET_TYPE --detrend $URL_DET --input_uri $URI_IN
+    command detrend_create_resid.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --class_id $CLASS_ID --det_type $DET_TYPE --detrend $URL_DET --input_uri $URI_IN
   end
 
@@ -119 +142 @@
   # operation times out?
   task.exit    timeout
-    queuepush DetrendResidImfiles -replace -key 1:3:4:5 "FAIL $options:0"
-  end
-end
+    queuepush DetrendResidImfiles -replace -key 1:3:4:5 "TIMEOUT $options:0"
+  end
+end
+
+# select images ready for copy 
+# new entries are added to queue DetrendResidExp
+# compare the new list with the ones already selected
+task           dettool.residexp.load
+  command      dettool -toresidexp -simple
+  host         local
+
+  periods      -poll 1
+  periods      -exec 5
+  periods      -timeout 5
+
+  stdout dettool.residexp.log
+  stderr dettool.residexp.log
+
+  # success
+  task.exit    0
+    # XXX is it necessary for these to be local?
+    # drop after this task macro is done?
+    local i Nqueue
+
+    # compare output with newImage queue
+    # only add entries which don't exist in queue
+    queuesize stdout -var Nqueue
+    for i 0 $Nqueue
+      queuepop stdout -var line
+      list word -split $line
+      $DET_ID   = $word:0
+      $ITER     = $word:1
+      $DET_TYPE = $word:2
+      $EXP_KEY  = $word:3
+      $INCLUDE  = $word:4
+      queuepush DetrendResidExp -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_KEY $INCLUDE"
+    end
+
+    # delete existing entries which are DONE
+    queuedrop DetrendResidExp -key 0 DONE
+  end
+
+  # locked list
+  task.exit    1
+    echo       "new.images: exec failure"
+  end
+
+  # default exit status
+  task.exit    default
+    echo       "new.images: unknown exit status: $EXIT"
+  end
+
+  # operation times out?
+  task.exit    timeout
+    echo       "new.images: timeout"
+  end
+end
+
+# copy new images, sending job to desired host
+task           dettool.residexp.process
+  periods      -poll 0.5
+  periods      -exec 5.0
+  periods      -timeout 15
+
+  task.exec
+    queuesize DetrendResidExp -var N
+    if ($N == 0) break
+    if ($network == 0) break
+    
+    # look for new images on the internal DetrendResidExp queue
+    # caution with these 'if' statements: syntax errors 
+    # will make the task fail without given a good status
+    queuepop DetrendResidExp -var line -key 0 NEW
+    if ("$line" == "NULL") break
+
+    strpop line state
+    queuepush DetrendResidExp -replace -key 1:2:4 "RUN $line"
+
+    ## the DetrendResidExp queue contains:
+    ## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN
+    ## NOTE: the queue has the STATE prepended 
+    list word -split $line
+    $DET_ID   = $word:0
+    $ITER     = $word:1
+    $DET_TYPE = $word:2
+    $EXP_KEY  = $word:3
+    $INCLUDE  = $word:4
+
+    stdout $EXP_KEY.detresid.$DET_ID.$ITER.log
+    stderr $EXP_KEY.detresid.$DET_ID.$ITER.log
+
+    # specify choice of remote host:
+    if ($parallel)
+      host anyhost
+    else
+      host local
+    end
+
+    # create example job options as a demonstration
+    options "$line"
+    echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --det_type $DET_TYPE
+    command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --det_type $DET_TYPE
+  end
+
+  # success
+  task.exit 0
+    queuepush DetrendResidExp -replace -key 1:2:4 "DONE $options:0"
+  end
+
+  # default exit status
+  task.exit    default
+    queuepush DetrendResidExp -replace -key 1:2:4 "FAIL $options:0"
+  end
+
+  # operation times out?
+  task.exit    timeout
+    queuepush DetrendResidExp -replace -key 1:2:4 "TIMEOUT $options:0"
+  end
+end