Index: branches/eam_branch_00/ippTasks/detrend.resid.pro =================================================================== --- branches/eam_branch_00/ippTasks/detrend.resid.pro (revision 10761) +++ branches/eam_branch_00/ippTasks/detrend.resid.pro (revision 11011) @@ -1,19 +1,4 @@ - ## this file contains the tasks for running the detrend processing stage -# dettool -toresidimfile -simple -# 0 bias 1 850131b mc o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.565582 -# ITER TYPE DET_ID EXP_TAG CLASS_ID URI-IN - -# detrend_create_resid.pl 1 0 850131b mc bias bias_mc_1_0.fit o_850131b.mc.fit 850131b -# detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG) -# URL-DET: $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit - -## these tasks use the queue DetrendResidImfiles -## the DetrendResidImfiles queue contains: -## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN - -queueinit DetrendResidImfiles -queueinit DetrendResidExposures if ($?network == 0) $network = 1 @@ -22,8 +7,4 @@ $parallel = 0 end - -$DetrendResidImfileFail = 0 -$DetrendResidExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -34,12 +15,14 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR +book init DetrendResidImfiles +book init DetrendResidExposures + macro detresid.status - queueprint DetrendResidImfiles - queueprint DetrendResidExposures - echo "DetrendResidImfileFail: $DetrendResidImfileFail" - echo "DetrendResidExpFail: $DetrendResidExpFail" + book listbook DetrendResidImfiles + book listbook DetrendResidExposures end @@ -58,41 +41,19 @@ task.exec - command dettool -toresidimfile -simple -limit {$DetrendResidImfileFail + 20} + command dettool -toresidimfile -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $CLASS_ID = $word:5 - $URI = $word:6 - $DET_URI = $word:7 - $CAMERA = $word:8 - ## must include the detID and iter in the key (if we have two running at the same time...) - ## Also needed to be in the output file (or path) - queuepush DetrendResidImfiles -uniq -key 1:2:5:6 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $CLASS_ID $URI $DET_URI $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout DetrendResidImfiles -key det_id:iter:exp_tag:class_id -uniq + if ($VERBOSE > 2) + book listbook DetrendResidImfiles end # delete existing entries which are DONE - queuedrop DetrendResidImfiles -key 0 DONE - end - - # locked list + book delpage DetrendResidImfiles -key state DONE + end + # default exit status task.exit default @@ -113,30 +74,29 @@ task.exec - queuesize DetrendResidImfiles -var N + book npages DetrendResidImfiles -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendResidImfiles queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendResidImfiles -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendResidImfiles -replace -key 1:2:5:6 "RUN $line" - - ## the DetrendResidImfiles queue contains: - ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $CLASS_ID = $word:5 - $URI = $word:6 - $DET_URI = $word:7 - $CAMERA = $word:8 + # look for new images in DetrendResidImfiles + book getpage DetrendResidImfiles 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword DetrendResidImfiles $pageName state RUN + book getword DetrendResidImfiles $pageName DET_ID -var DET_ID + book getword DetrendResidImfiles $pageName ITER -var ITER + book getword DetrendResidImfiles $pageName DET_TYPE -var DET_TYPE + book getword DetrendResidImfiles $pageName MODE -var MODE + book getword DetrendResidImfiles $pageName EXP_TAG -var EXP_TAG + book getword DetrendResidImfiles $pageName CLASS_ID -var CLASS_ID + book getword DetrendResidImfiles $pageName URI -var URI + book getword DetrendResidImfiles $pageName DET_URI -var DET_URI + book getword DetrendResidImfiles $pageName CAMERA -var CAMERA + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end $word = `basename $EXP_TAG | tr '.' ' '` @@ -147,4 +107,96 @@ exec mkdir -p $LOGSUBDIR/$base + # save the pageName for future reference below + options $pageName + + # create command + if ($VERBOSE > 1) + echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE + end + command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE + end + + # success + task.exit 0 + book setword DetrendResidImfiles $options:0 state DONE + end + + # default exit status + task.exit default + showcommand failure + book setword DetrendResidImfiles $options:0 state FAIL + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword DetrendResidImfiles $options:0 state TIMEOUT + end +end + +# select images ready for copy +# new entries are added to DetrendResidExposures +# compare the new list with the ones already selected +task dettool.residexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/detresid.exp.log + stderr $LOGSUBDIR/detresid.exp.log + + task.exec + command dettool -toresidexp -limit 20 + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout DetrendResidExposures -key det_id:iter:exp_tag -uniq + if ($VERBOSE > 2) + book listbook DetrendResidExposures + end + + # delete existing entries which are DONE + book delpage DetrendResidExposures -key state DONE + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task dettool.residexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages DetrendResidExposures -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in DetrendResidExposures + book getpage DetrendResidExposures 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword DetrendResidExposures $pageName state RUN + book getword DetrendResidExposures $pageName DET_ID -var DET_ID + book getword DetrendResidExposures $pageName ITER -var ITER + book getword DetrendResidExposures $pageName DET_TYPE -var DET_TYPE + book getword DetrendResidExposures $pageName MODE -var MODE + book getword DetrendResidExposures $pageName EXP_TAG -var EXP_TAG + book getword DetrendResidExposures $pageName INCLUDE -var INCLUDE + book getword DetrendResidExposures $pageName CAMERA -var CAMERA + # specify choice of remote host: if ($parallel) @@ -154,141 +206,16 @@ end - # create example job options as a demonstration - options "$line" - # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG ???) - # need to parse the subdir path for the detrend URL - # $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits - echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE - command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE - end - - # success - task.exit 0 - queuepush DetrendResidImfiles -replace -key 1:2:5:6 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendResidImfiles -replace -key 1:2:5:6 "FAIL $options:0" - $DetrendResidImfileFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendResidImfiles -replace -key 1:2:6:6 "TIMEOUT $options:0" - $DetrendResidImfileFail ++ - end -end - -# select images ready for copy -# new entries are added to queue DetrendResidExposures -# compare the new list with the ones already selected -task dettool.residexp.load - host local - - periods -poll $loadpoll - periods -exec $loadexec - periods -timeout 30 - - stdout $LOGSUBDIR/detresid.exp.log - stderr $LOGSUBDIR/detresid.exp.log - - task.exec - command dettool -toresidexp -simple -limit {$DetrendResidExpFail + 20} - end - - # success - task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $INCLUDE = $word:5 - $CAMERA = $word:6 - queuepush DetrendResidExposures -uniq -key 1:2:5 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $INCLUDE $CAMERA" - end - - # delete existing entries which are DONE - queuedrop DetrendResidExposures -key 0 DONE - end - - # locked list - # default exit status - task.exit default - showcommand failure - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# copy new images, sending job to desired host -task dettool.residexp.run - periods -poll $runpoll - periods -exec $runexec - periods -timeout 30 - - task.exec - queuesize DetrendResidExposures -var N - if ($N == 0) break - if ($network == 0) break - - # look for new images on the internal DetrendResidExposures queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendResidExposures -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendResidExposures -replace -key 1:2:5 "RUN $line" - - ## the DetrendResidExposures queue contains: - ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $INCLUDE = $word:5 - $CAMERA = $word:6 - # XXX this is still a lame rule $word = `basename $EXP_TAG | tr '.' ' '` list word -split $word $base = $word:0 - stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log exec mkdir -p $LOGSUBDIR/$base - # specify choice of remote host: - if ($parallel) - host anyhost - else - host local - end + # save the pageName for future reference below + options $pageName # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA @@ -299,19 +226,17 @@ # success task.exit 0 - queuepush DetrendResidExposures -replace -key 1:2:5 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendResidExposures -replace -key 1:2:5 "FAIL $options:0" - $DetrendResidExpFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendResidExposures -replace -key 1:2:5 "TIMEOUT $options:0" - $DetrendResidExpFail ++ - end -end + book setword DetrendResidExposures $options:0 state DONE + end + + # default exit status + task.exit default + showcommand failure + book setword DetrendResidExposures $options:0 state FAIL + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword DetrendResidExposures $options:0 state TIMEOUT + end +end