Index: trunk/ippTasks/detrend.resid.pro =================================================================== --- trunk/ippTasks/detrend.resid.pro (revision 10745) +++ trunk/ippTasks/detrend.resid.pro (revision 11210) @@ -1,19 +1,5 @@ - ## this file contains the tasks for running the detrend processing stage - -# dettool -toresidimfile -simple -# 0 bias 1 850131b mc o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.565582 -# ITER TYPE DET_ID EXP_TAG CLASS_ID URI-IN - -# detrend_create_resid.pl 1 0 850131b mc bias bias_mc_1_0.fit o_850131b.mc.fit 850131b -# detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG) -# URL-DET: $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit - -## these tasks use the queue DetrendResidImfiles -## the DetrendResidImfiles queue contains: -## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN - -queueinit DetrendResidImfiles -queueinit DetrendResidExposures +## these tasks use the books detPendingResidImfile and detPendingResidExp + if ($?network == 0) $network = 1 @@ -22,8 +8,4 @@ $parallel = 0 end - -$DetrendResidImfileFail = 0 -$DetrendResidExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -34,16 +16,18 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR +book init detPendingResidImfile +book init detPendingResidExp + macro detresid.status - queueprint DetrendResidImfiles - queueprint DetrendResidExposures - echo "DetrendResidImfileFail: $DetrendResidImfileFail" - echo "DetrendResidExpFail: $DetrendResidExpFail" + book listbook detPendingResidImfile + book listbook detPendingResidExp end # select images ready for copy -# new entries are added to queue DetrendResidImfiles +# new entries are added to detPendingResidImfile # compare the new list with the ones already selected task dettool.resid.load @@ -54,45 +38,23 @@ periods -timeout 30 - stdout $LOGSUBDIR/detresid.log + stdout NULL stderr $LOGSUBDIR/detresid.log task.exec - command dettool -toresidimfile -simple -limit {$DetrendResidImfileFail + 20} + command dettool -toresidimfile -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $CLASS_ID = $word:5 - $URI = $word:6 - $DET_URI = $word:7 - $CAMERA = $word:8 - ## must include the detID and iter in the key (if we have two running at the same time...) - ## Also needed to be in the output file (or path) - queuepush DetrendResidImfiles -uniq -key 1:2:5:6 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $CLASS_ID $URI $DET_URI $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout detPendingResidImfile -key det_id:iteration:exp_tag:class_id -uniq + if ($VERBOSE > 2) + book listbook detPendingResidImfile end # delete existing entries which are DONE - queuedrop DetrendResidImfiles -key 0 DONE - end - - # locked list + book delpage detPendingResidImfile -key state DONE + end + # default exit status task.exit default @@ -113,37 +75,22 @@ task.exec - queuesize DetrendResidImfiles -var N + book npages detPendingResidImfile -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendResidImfiles queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendResidImfiles -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendResidImfiles -replace -key 1:2:5:6 "RUN $line" - - ## the DetrendResidImfiles queue contains: - ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $CLASS_ID = $word:5 - $URI = $word:6 - $DET_URI = $word:7 - $CAMERA = $word:8 - - $word = `basename $EXP_TAG | tr '.' ' '` - list word -split $word - $base = $word:0 - stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log - stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log - exec mkdir -p $LOGSUBDIR/$base + # look for new images in detPendingResidImfile + book getpage detPendingResidImfile 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword detPendingResidImfile $pageName state RUN + book getword detPendingResidImfile $pageName det_id -var DET_ID + book getword detPendingResidImfile $pageName iteration -var ITERATION + book getword detPendingResidImfile $pageName det_type -var DET_TYPE + book getword detPendingResidImfile $pageName mode -var MODE + book getword detPendingResidImfile $pageName exp_tag -var EXP_TAG + book getword detPendingResidImfile $pageName class_id -var CLASS_ID + book getword detPendingResidImfile $pageName uri -var URI + book getword detPendingResidImfile $pageName det_uri -var DET_URI + book getword detPendingResidImfile $pageName camera -var CAMERA # specify choice of remote host: @@ -154,35 +101,36 @@ end - # create example job options as a demonstration - options "$line" - # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG ???) - # need to parse the subdir path for the detrend URL - # $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits - echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE - command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE - end - - # success - task.exit 0 - queuepush DetrendResidImfiles -replace -key 1:2:5:6 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendResidImfiles -replace -key 1:2:5:6 "FAIL $options:0" - $DetrendResidImfileFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendResidImfiles -replace -key 1:2:6:6 "TIMEOUT $options:0" - $DetrendResidImfileFail ++ + # XXX use ipp_filename.pl to lookup output file names + $word = `basename $EXP_TAG | tr '.' ' '` + list word -split $word + $base = $word:0 + stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.$CLASS_ID.log + stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.$CLASS_ID.log + exec mkdir -p $LOGSUBDIR/$base + + # save the pageName for future reference below + options $pageName + + # create command + if ($VERBOSE > 1) + echo command detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE + end + command detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE + end + + # default exit status + task.exit default + process_exit detPendingResidImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidImfile $options:0 state TIMEOUT end end # select images ready for copy -# new entries are added to queue DetrendResidExposures +# new entries are added to detPendingResidExp # compare the new list with the ones already selected task dettool.residexp.load @@ -193,41 +141,23 @@ periods -timeout 30 - stdout $LOGSUBDIR/detresid.exp.log + stdout NULL stderr $LOGSUBDIR/detresid.exp.log task.exec - command dettool -toresidexp -simple -limit {$DetrendResidExpFail + 20} + command dettool -toresidexp -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $INCLUDE = $word:5 - $CAMERA = $word:6 - queuepush DetrendResidExposures -uniq -key 1:2:5 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $INCLUDE $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout detPendingResidExp -key det_id:iteration:exp_tag -uniq + if ($VERBOSE > 2) + book listbook detPendingResidExp end # delete existing entries which are DONE - queuedrop DetrendResidExposures -key 0 DONE - end - - # locked list + book delpage detPendingResidExp -key state DONE + end + # default exit status task.exit default @@ -248,37 +178,20 @@ task.exec - queuesize DetrendResidExposures -var N + book npages detPendingResidExp -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendResidExposures queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendResidExposures -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendResidExposures -replace -key 1:2:5 "RUN $line" - - ## the DetrendResidExposures queue contains: - ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $INCLUDE = $word:5 - $CAMERA = $word:6 - - # XXX this is still a lame rule - $word = `basename $EXP_TAG | tr '.' ' '` - list word -split $word - $base = $word:0 - - stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log - stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log - exec mkdir -p $LOGSUBDIR/$base + # look for new images in detPendingResidExp + book getpage detPendingResidExp 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword detPendingResidExp $pageName state RUN + book getword detPendingResidExp $pageName det_id -var DET_ID + book getword detPendingResidExp $pageName iteration -var ITERATION + book getword detPendingResidExp $pageName det_type -var DET_TYPE + book getword detPendingResidExp $pageName mode -var MODE + book getword detPendingResidExp $pageName exp_tag -var EXP_TAG + book getword detPendingResidExp $pageName include -var INCLUDE + book getword detPendingResidExp $pageName camera -var CAMERA # specify choice of remote host: @@ -289,29 +202,31 @@ end + # XXX use ipp_filename.pl to lookup output file names + $word = `basename $EXP_TAG | tr '.' ' '` + list word -split $word + $base = $word:0 + stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.log + stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.log + exec mkdir -p $LOGSUBDIR/$base + + # save the pageName for future reference below + options $pageName + # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) - echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA - end - command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA - end - - # success - task.exit 0 - queuepush DetrendResidExposures -replace -key 1:2:5 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendResidExposures -replace -key 1:2:5 "FAIL $options:0" - $DetrendResidExpFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendResidExposures -replace -key 1:2:5 "TIMEOUT $options:0" - $DetrendResidExpFail ++ - end -end + echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA + end + command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA + end + + # default exit status + task.exit default + process_exit detPendingResidExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidExp $options:0 state TIMEOUT + end +end