Index: unk/ippTasks/detrend.assess.pro =================================================================== --- /trunk/ippTasks/detrend.assess.pro (revision 9103) +++ (revision ) @@ -1,118 +1,0 @@ - -## this file contains the tasks for running the detrend processing stage - -# dettool -toresidexp -simple -# 1 0 bias 850131b T -# DET_ID ITER TYPE EXP_KEY INCLUDE - -# detrend_reject_imfile.pl 1 0 850131b bias -# detrend_reject_imfile.pl DET_ID ITER EXP_KEY TYPE - -## these tasks use the queue DetrendResidExp -## the DetrendResidExp queue contains: -## STATE DET_ID ITER TYPE EXP_KEY INCLUDE - -# select images ready for copy -# new entries are added to queue DetrendResidExp -# compare the new list with the ones already selected -task dettool.assess.load - command dettool -toresidexp -simple - host local - - periods -poll 1 - periods -exec 5 - periods -timeout 30 - - # success - task.exit 0 - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $TYPE = $word:2 - $EXP_KEY = $word:3 - $INCLUDE = $word:4 - queuepush DetrendResidExp -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $EXP_KEY $INCLUDE" - end - - # delete existing entries which are DONE - queuedrop DetrendResidExp -key 0 DONE - end - - # locked list - task.exit 1 - echo "new.images: exec failure" - end - - # default exit status - task.exit default - echo "new.images: unknown exit status: $EXIT" - end - - # operation times out? - task.exit timeout - echo "new.images: timeout" - end -end - -# copy new images, sending job to desired host -task dettool.resid.process - periods -poll 0.5 - periods -exec 1.0 - periods -timeout 5 - - task.exec - queuesize DetrendResidExp -var N - if ($N == 0) break - if ($network == 0) break - - # look for new images on the internal DetrendResidExp queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendResidExp -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendResidExp -replace -key 1:2:4 "RUN $line" - - ## the DetrendResidExp queue contains: - ## STATE DET_ID ITER TYPE EXP_KEY INCLUDE - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:1 - $ITER = $word:2 - $TYPE = $word:3 - $EXP_KEY = $word:4 - $INCLUDE = $word:5 - - # specify choice of remote host: - host anyhost - - # create example job options as a demonstration - options "$line" - # detrend_reject_imfile.pl 1 0 850131b bias - # detrend_reject_imfile.pl DET_ID ITER EXP_KEY TYPE - command detrend_reject_imfile.pl $DET_ID $ITER $EXP_KEY $TYPE - end - - # success - task.exit 0 - queuepush DetrendResidExp -replace -key 1:2:4 "DONE $options:0" - end - - # default exit status - task.exit default - queuepush DetrendResidExp -replace -key 1:2:4 "FAIL $options:0" - end - - # operation times out? - task.exit timeout - queuepush DetrendResidExp -replace -key 1:2:4 "FAIL $options:0" - end -end Index: /trunk/ippTasks/detrend.norm.pro =================================================================== --- /trunk/ippTasks/detrend.norm.pro (revision 9103) +++ /trunk/ippTasks/detrend.norm.pro (revision 9104) @@ -2,16 +2,31 @@ ## this file contains the tasks for running the detrend normalization stage -## these tasks use the queue DetrendRunToNormalize +## these tasks use the queue DetrendNormalize +## the DetrendNormalize queue contains: +## STATE DET_ID ITER + +queueinit DetrendNormalize +queueinit DetrendNormStat +queueinit DetrendNormalizeExposures +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end # select images ready for copy -# new entries are added to queue DetrendRunToNormalize +# new entries are added to queue DetrendNormalize # compare the new list with the ones already selected task dettool.norm.load - command dettool -stacked -runs + command dettool -tonormalize -simple host local periods -poll 1 periods -exec 5 - periods -timeout 30 + periods -timeout 5 + + stdout dettool.norm.log + stderr dettool.norm.log # success @@ -26,24 +41,20 @@ for i 0 $Nqueue queuepop stdout -var line - queuepush DetrendRunToNormalize -uniq -key 1:2:4 "NEW $line" + queuepush DetrendNormalize -uniq -key 1:2 "NEW $line" end # delete existing entries which are DONE - queuedrop DetrendRunToNormalize -key 0 DONE + queuedrop DetrendNormalize -key 0 DONE end # locked list - task.exit 1 - echo "new.images: exec failure" - end - - # default exit status - task.exit default - echo "new.images: unknown exit status: $EXIT" - end - - # operation times out? - task.exit timeout - echo "new.images: timeout" + # default exit status + task.exit default + echo "detrend.norm: failure" + end + + # operation times out? + task.exit timeout + echo "detrend.norm: timeout" end end @@ -52,59 +63,272 @@ task dettool.norm.process periods -poll 0.5 - periods -exec 1.0 - periods -timeout 5 + periods -exec 5.0 + periods -timeout 30 task.exec - queuesize DetrendRunToNormalize -var N + queuesize DetrendNormalize -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendRunToNormalize queue + # look for new images on the internal DetrendNormalize queue # caution with these 'if' statements: syntax errors # will make the task fail without given a good status - queuepop DetrendRunToNormalize -var line -key 0 NEW + queuepop DetrendNormalize -var line -key 0 NEW if ("$line" == "NULL") break strpop line state - queuepush DetrendRunToNormalize -replace -key 0 "RUN $line" + queuepush DetrendNormalize -replace -key 1:2 "RUN $line" + + # the DetrendNormalize queue contains: + # STATE DET_ID ITER + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + + # XXX we need to get CAMERA and TYPE from this query + # stdout $CAMERA.$TYPE.$DET_ID.$ITER.log + # stderr $CAMERA.$TYPE.$DET_ID.$ITER.log + stdout isp.flat.$DET_ID.$ITER.log + stderr isp.flat.$DET_ID.$ITER.log # specify choice of remote host: - # 'anyhost' specified random remote host - # if we needed to specify a host explicitly, use a command like - # one of the following. the -required forces execution on the - # named host, and blocks the task until the host is available - # host kiawe - # host alala -required - host anyhost - - # choose the default recipe appropriate to this type of file - $recipe = `get.recipe $type` + if ($parallel) + host anyhost + else + host local + end # create example job options as a demonstration options "$line" - - # we can put the dettool and ppMerge commands here, or wrap them in a single script - # $urilist = `dettool -stacked -det_id $det_id -iteration $iteration -chip" - # $uri_out = ??? - # command ppNorm $uri_out $uri $uri ... -recipe PPMERGE $recipe ??? - command ppNorm $uri_out -det_id $det_id -iteration $iteration -recipe PPMERGE $recipe + echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER + command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER end # success task.exit 0 - # XXX need to parse the options to identify the values below (if not part of cmdline?) - exec dettool -addstacked -det_id $det_id -iteration $iteration -class_id $class_id -procrecipe $recipe - queuepush DetrendRunToNormalize -replace -key 1:2:4 "DONE $options:0" - end - - # default exit status - task.exit default - queuepush DetrendRunToNormalize -replace -key 0 "FAIL $options:0" - end - - # operation times out? - task.exit timeout - queuepush DetrendRunToNormalize -replace -key 0 "FAIL $options:0" - end -end + queuepush DetrendNormalize -replace -key 1:2 "DONE $options:0" + end + + # default exit status + task.exit default + queuepush DetrendNormalize -replace -key 1:2 "FAIL $options:0" + end + + # operation times out? + task.exit timeout + queuepush DetrendNormalize -replace -key 1:2 "TIMEOUT $options:0" + end +end + +# select images ready for copy +# new entries are added to queue DetrendNormStat +# compare the new list with the ones already selected +task dettool.normstat.load + command dettool -normstat -simple + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 5 + + stdout dettool.normstat.log + stderr dettool.normstat.log + + # success + task.exit 0 + # XXX is it necessary for these to be local? + # drop after this task macro is done? + local i Nqueue + + # compare output with newImage queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + queuepush DetrendNormStat -uniq -key 1:2 "NEW $line" + end + + # delete existing entries which are DONE + queuedrop DetrendNormStat -key 0 DONE + end + + # locked list + # default exit status + task.exit default + echo "detrend.norm: failure" + end + + # operation times out? + task.exit timeout + echo "detrend.norm: timeout" + end +end + +# copy new images, sending job to desired host +task dettool.normstat.process + periods -poll 0.5 + periods -exec 5.0 + periods -timeout 30 + + task.exec + queuesize DetrendNormStat -var N + + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendNormStat queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendNormStat -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendNormStat -replace -key 1:2 "RUN $line" + + # the DetrendNormStat queue contains: + # STATE DET_ID ITER + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + + # XXX we need to get CAMERA and TYPE from this query + # stdout $CAMERA.$TYPE.$DET_ID.$ITER.log + # stderr $CAMERA.$TYPE.$DET_ID.$ITER.log + stdout isp.flat.$DET_ID.$ITER.log + stderr isp.flat.$DET_ID.$ITER.log + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER + command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER + end + + # success + task.exit 0 + queuepush DetrendNormStat -replace -key 1:2 "DONE $options:0" + end + + # default exit status + task.exit default + queuepush DetrendNormStat -replace -key 1:2 "FAIL $options:0" + end + + # operation times out? + task.exit timeout + queuepush DetrendNormStat -replace -key 1:2 "TIMEOUT $options:0" + end +end + +# select images ready for copy +# new entries are added to queue DetrendNormalizeExposures +# compare the new list with the ones already selected +task dettool.normexp.load + command dettool -tonormalizedexp -simple + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 5 + + stdout dettool.normexp.log + stderr dettool.normexp.log + + # success + task.exit 0 + # XXX is it necessary for these to be local? + # drop after this task macro is done? + local i Nqueue + + # compare output with newImage queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + queuepush DetrendNormalizeExposures -uniq -key 1:2 "NEW $line" + end + + # delete existing entries which are DONE + queuedrop DetrendNormalizeExposures -key 0 DONE + end + + # locked list + # default exit status + task.exit default + echo "detrend.norm: failure" + end + + # operation times out? + task.exit timeout + echo "detrend.norm: timeout" + end +end + +# copy new images, sending job to desired host +task dettool.normexp.process + periods -poll 0.5 + periods -exec 5.0 + periods -timeout 30 + + task.exec + queuesize DetrendNormalizeExposures -var N + + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendNormalizeExposures queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendNormalizeExposures -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendNormalizeExposures -replace -key 1:2 "RUN $line" + + # the DetrendNormalizeExposures queue contains: + # STATE DET_ID ITER + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + + # XXX we need to get CAMERA and TYPE from this query + # stdout $CAMERA.$TYPE.$DET_ID.$ITER.log + # stderr $CAMERA.$TYPE.$DET_ID.$ITER.log + stdout isp.flat.$DET_ID.$ITER.log + stderr isp.flat.$DET_ID.$ITER.log + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER + command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER + end + + # success + task.exit 0 + queuepush DetrendNormalizeExposures -replace -key 1:2 "DONE $options:0" + end + + # default exit status + task.exit default + queuepush DetrendNormalizeExposures -replace -key 1:2 "FAIL $options:0" + end + + # operation times out? + task.exit timeout + queuepush DetrendNormalizeExposures -replace -key 1:2 "TIMEOUT $options:0" + end +end Index: /trunk/ippTasks/detrend.process.pro =================================================================== --- /trunk/ippTasks/detrend.process.pro (revision 9103) +++ /trunk/ippTasks/detrend.process.pro (revision 9104) @@ -9,9 +9,10 @@ # detrend_process.pl 1 850131b mc bias 850131b.fits 850131b -## these tasks use the queue DetrendImfilesToProcess -## the DetrendImfilesToProcess queue contains: +## these tasks use the queue DetrendProcessImfiles +## the DetrendProcessImfiles queue contains: ## STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI -queueinit DetrendImfilesToProcess +queueinit DetrendProcessImfiles +queueinit DetrendProcessExposures if ($?network == 0) $network = 1 @@ -22,7 +23,7 @@ # select images ready for copy -# new entries are added to queue DetrendImfilesToProcess +# new entries are added to queue DetrendProcessImfiles # compare the new list with the ones already selected -task dettool.raw.load +task dettool.process.load command dettool -raw -simple host local @@ -53,29 +54,24 @@ $CLASS_ID = $word:4 $URI = $word:5 - queuepush DetrendImfilesToProcess -uniq -key 1:3:5 "NEW $DET_ID $TYPE $EXP_KEY $CLASS $CLASS_ID $URI" + queuepush DetrendProcessImfiles -uniq -key 1:3:5 "NEW $DET_ID $TYPE $EXP_KEY $CLASS $CLASS_ID $URI" end # delete existing entries which are DONE - queuedrop DetrendImfilesToProcess -key 0 DONE - end - - # locked list - task.exit 1 - echo "new.images: exec failure" - end - - # default exit status - task.exit default - echo "new.images: unknown exit status: $EXIT" - end - - # operation times out? - task.exit timeout - echo "new.images: timeout" + queuedrop DetrendProcessImfiles -key 0 DONE + end + + # error + task.exit default + echo "detrend.process: failure" + end + + # operation times out? + task.exit timeout + echo "detrend.process: timeout" end end # copy new images, sending job to desired host -task dettool.raw.process +task dettool.process.run periods -poll 0.5 periods -exec 2.0 @@ -83,18 +79,18 @@ task.exec - queuesize DetrendImfilesToProcess -var N + queuesize DetrendProcessImfiles -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendImfilesToProcess queue + # look for new images on the internal DetrendProcessImfiles queue # caution with these 'if' statements: syntax errors # will make the task fail without given a good status - queuepop DetrendImfilesToProcess -var line -key 0 NEW + queuepop DetrendProcessImfiles -var line -key 0 NEW if ("$line" == "NULL") break strpop line state - queuepush DetrendImfilesToProcess -replace -key 1:3:5 "RUN $line" - - ## the DetrendImfilesToProcess queue contains: + queuepush DetrendProcessImfiles -replace -key 1:3:5 "RUN $line" + + ## the DetrendProcessImfiles queue contains: # STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI ## NOTE: the queue has the STATE prepended @@ -119,21 +115,128 @@ # create example job options as a demonstration options "$line" - echo command detrend_process.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI - command detrend_process.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI + echo command detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI + command detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI end # success task.exit 0 - queuepush DetrendImfilesToProcess -replace -key 1:3:5 "DONE $options:0" + queuepush DetrendProcessImfiles -replace -key 1:3:5 "DONE $options:0" end # default exit status task.exit default - queuepush DetrendImfilesToProcess -replace -key 1:3:5 "FAIL $options:0" - end - - # operation times out? - task.exit timeout - queuepush DetrendImfilesToProcess -replace -key 1:3:5 "TIMEOUT $options:0" - end -end + queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0" + end + + # operation times out? + task.exit timeout + queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0" + end +end + +# select images ready for copy +# new entries are added to queue DetrendProcessExposures +# compare the new list with the ones already selected +task dettool.processexp.load + command dettool -toprocessdexp -simple + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + + stdout dettool.processexp.log + stderr dettool.processexp.log + + # success + task.exit 0 + # XXX is it necessary for these to be local? + # drop after this task macro is done? + local i Nqueue + + # compare output with newImage queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + list word -split $line + $DET_ID = $word:0 + $TYPE = $word:1 + $EXP_KEY = $word:2 + queuepush DetrendProcessExposures -uniq -key 1:3 "NEW $DET_ID $TYPE $EXP_KEY" + end + + # delete existing entries which are DONE + queuedrop DetrendProcessExposures -key 0 DONE + end + + # error + task.exit default + echo "detrend.process: failure" + end + + # operation times out? + task.exit timeout + echo "detrend.process: timeout" + end +end + +# copy new images, sending job to desired host +task dettool.processexp.run + periods -poll 0.5 + periods -exec 2.0 + periods -timeout 60 + + task.exec + queuesize DetrendProcessExposures -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendProcessExposures queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendProcessExposures -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendProcessExposures -replace -key 1:3:5 "RUN $line" + + ## the DetrendProcessExposures queue contains: + # STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI + ## NOTE: the queue has the STATE prepended + list word -split $line + $DET_ID = $word:0 + $TYPE = $word:1 + $EXP_KEY = $word:2 + + stdout $EXP_KEY.detproc.$CLASS_ID.log + stderr $EXP_KEY.detproc.$CLASS_ID.log + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + echo command detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_KEY --det_type bias + command detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_KEY --det_type bias + end + + # success + task.exit 0 + queuepush DetrendProcessExposures -replace -key 1:3:5 "DONE $options:0" + end + + # default exit status + task.exit default + queuepush DetrendProcessExposures -replace -key 1:3:5 "FAIL $options:0" + end + + # operation times out? + task.exit timeout + queuepush DetrendProcessExposures -replace -key 1:3:5 "TIMEOUT $options:0" + end +end Index: /trunk/ippTasks/detrend.reject.pro =================================================================== --- /trunk/ippTasks/detrend.reject.pro (revision 9103) +++ /trunk/ippTasks/detrend.reject.pro (revision 9104) @@ -13,5 +13,5 @@ ## STATE DET_ID ITER TYPE -queueinit DetrendRejectExp +queueinit DetrendRejectExposure if ($?network == 0) $network = 1 @@ -22,5 +22,5 @@ # select images ready for copy -# new entries are added to queue DetrendRejectExp +# new entries are added to queue DetrendRejectExposure # compare the new list with the ones already selected task dettool.reject.load @@ -48,29 +48,24 @@ $ITER = $word:1 $DET_TYPE = $word:2 - queuepush DetrendRejectExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE" + queuepush DetrendRejectExposure -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE" end # delete existing entries which are DONE - queuedrop DetrendRejectExp -key 0 DONE + queuedrop DetrendRejectExposure -key 0 DONE end # locked list - task.exit 1 - echo "new.images: exec failure" - end - - # default exit status task.exit default - echo "new.images: unknown exit status: $EXIT" + echo "detrend.reject: failure" end # operation times out? task.exit timeout - echo "new.images: timeout" + echo "detrend.reject: timeout" end end # copy new images, sending job to desired host -task dettool.reject.process +task dettool.reject.run periods -poll 0.5 periods -exec 1.0 @@ -78,18 +73,18 @@ task.exec - queuesize DetrendRejectExp -var N + queuesize DetrendRejectExposure -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendRejectExp queue + # look for new images on the internal DetrendRejectExposure queue # caution with these 'if' statements: syntax errors # will make the task fail without given a good status - queuepop DetrendRejectExp -var line -key 0 NEW + queuepop DetrendRejectExposure -var line -key 0 NEW if ("$line" == "NULL") break strpop line state - queuepush DetrendRejectExp -replace -key 1:2 "RUN $line" + queuepush DetrendRejectExposure -replace -key 1:2 "RUN $line" - ## the DetrendRejectExp queue contains: + ## the DetrendRejectExposure queue contains: ## STATE DET_ID ITER TYPE ## NOTE: the queue has the STATE prepended @@ -118,15 +113,15 @@ # success task.exit 0 - queuepush DetrendRejectExp -replace -key 1:2 "DONE $options:0" + queuepush DetrendRejectExposure -replace -key 1:2 "DONE $options:0" end # default exit status task.exit default - queuepush DetrendRejectExp -replace -key 1:2 "FAIL $options:0" + queuepush DetrendRejectExposure -replace -key 1:2 "FAIL $options:0" end # operation times out? task.exit timeout - queuepush DetrendRejectExp -replace -key 1:2 "TIMEOUT $options:0" + queuepush DetrendRejectExposure -replace -key 1:2 "TIMEOUT $options:0" end end Index: /trunk/ippTasks/detrend.resid.pro =================================================================== --- /trunk/ippTasks/detrend.resid.pro (revision 9103) +++ /trunk/ippTasks/detrend.resid.pro (revision 9104) @@ -15,5 +15,5 @@ queueinit DetrendResidImfiles -queueinit DetrendResidExp +queueinit DetrendResidExposures if ($?network == 0) $network = 1 @@ -65,21 +65,17 @@ # locked list - task.exit 1 - echo "new.images: exec failure" - end - - # default exit status - task.exit default - echo "new.images: unknown exit status: $EXIT" - end - - # operation times out? - task.exit timeout - echo "new.images: timeout" + # default exit status + task.exit default + echo "detrend.resid: failure" + end + + # operation times out? + task.exit timeout + echo "detrend.resid: timeout" end end # copy new images, sending job to desired host -task dettool.resid.process +task dettool.resid.run periods -poll 0.5 periods -exec 5.0 @@ -110,4 +106,5 @@ $CLASS_ID = $word:4 $URI_IN = $word:5 + $CAMERA = $word:6 stdout $EXP_KEY.detresid.$DET_ID.$ITER.$CLASS_ID.log @@ -124,6 +121,6 @@ options "$line" # detrend_create_resid.pl DET_ID ITER EXP_KEY CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_KEY ???) - $URL_DET = isp.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits - # $URL_DET = $CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits + # $URL_DET = isp.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits + $URL_DET = $CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits echo command detrend_create_resid.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --class_id $CLASS_ID --det_type $DET_TYPE --detrend $URL_DET --input_uri $URI_IN command detrend_create_resid.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --class_id $CLASS_ID --det_type $DET_TYPE --detrend $URL_DET --input_uri $URI_IN @@ -147,5 +144,5 @@ # select images ready for copy -# new entries are added to queue DetrendResidExp +# new entries are added to queue DetrendResidExposures # compare the new list with the ones already selected task dettool.residexp.load @@ -177,29 +174,25 @@ $EXP_KEY = $word:3 $INCLUDE = $word:4 - queuepush DetrendResidExp -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_KEY $INCLUDE" + queuepush DetrendResidExposures -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_KEY $INCLUDE" end # delete existing entries which are DONE - queuedrop DetrendResidExp -key 0 DONE + queuedrop DetrendResidExposures -key 0 DONE end # locked list - task.exit 1 - echo "new.images: exec failure" - end - - # default exit status - task.exit default - echo "new.images: unknown exit status: $EXIT" - end - - # operation times out? - task.exit timeout - echo "new.images: timeout" + # default exit status + task.exit default + echo "detrend.residexp: failure" + end + + # operation times out? + task.exit timeout + echo "detrend.residexp: timeout" end end # copy new images, sending job to desired host -task dettool.residexp.process +task dettool.residexp.run periods -poll 0.5 periods -exec 5.0 @@ -207,18 +200,18 @@ task.exec - queuesize DetrendResidExp -var N + queuesize DetrendResidExposures -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendResidExp queue + # look for new images on the internal DetrendResidExposures queue # caution with these 'if' statements: syntax errors # will make the task fail without given a good status - queuepop DetrendResidExp -var line -key 0 NEW + queuepop DetrendResidExposures -var line -key 0 NEW if ("$line" == "NULL") break strpop line state - queuepush DetrendResidExp -replace -key 1:2:4 "RUN $line" - - ## the DetrendResidExp queue contains: + queuepush DetrendResidExposures -replace -key 1:2:4 "RUN $line" + + ## the DetrendResidExposures queue contains: ## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN ## NOTE: the queue has the STATE prepended @@ -248,15 +241,15 @@ # success task.exit 0 - queuepush DetrendResidExp -replace -key 1:2:4 "DONE $options:0" - end - - # default exit status - task.exit default - queuepush DetrendResidExp -replace -key 1:2:4 "FAIL $options:0" - end - - # operation times out? - task.exit timeout - queuepush DetrendResidExp -replace -key 1:2:4 "TIMEOUT $options:0" - end -end + queuepush DetrendResidExposures -replace -key 1:2:4 "DONE $options:0" + end + + # default exit status + task.exit default + queuepush DetrendResidExposures -replace -key 1:2:4 "FAIL $options:0" + end + + # operation times out? + task.exit timeout + queuepush DetrendResidExposures -replace -key 1:2:4 "TIMEOUT $options:0" + end +end Index: /trunk/ippTasks/detrend.stack.pro =================================================================== --- /trunk/ippTasks/detrend.stack.pro (revision 9103) +++ /trunk/ippTasks/detrend.stack.pro (revision 9104) @@ -15,5 +15,5 @@ ## STATE DET_ID ITER TYPE CLASS_ID CAMERA -queueinit DetrendClassIDtoStack +queueinit DetrendStackClass if ($?network == 0) $network = 1 @@ -24,5 +24,5 @@ # select images ready for copy -# new entries are added to queue DetrendClassIDtoStack +# new entries are added to queue DetrendStackClass # compare the new list with the ones already selected task dettool.stack.load @@ -54,30 +54,25 @@ $CLASS_ID = $word:3 $CAMERA = $word:4 - echo queuepush DetrendClassIDtoStack -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA" - queuepush DetrendClassIDtoStack -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA" + echo queuepush DetrendStackClass -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA" + queuepush DetrendStackClass -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA" end # delete existing entries which are DONE - queuedrop DetrendClassIDtoStack -key 0 DONE + queuedrop DetrendStackClass -key 0 DONE end # locked list - task.exit 1 - echo "new.images: exec failure" - end - - # default exit status task.exit default - echo "new.images: unknown exit status: $EXIT" + echo "detrend.stack: failure" end # operation times out? task.exit timeout - echo "new.images: timeout" + echo "detrend.stack: timeout" end end # copy new images, sending job to desired host -task dettool.stack.process +task dettool.stack.run periods -poll 0.5 periods -exec 1.0 @@ -85,19 +80,19 @@ task.exec - queuesize DetrendClassIDtoStack -var N + queuesize DetrendStackClass -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendClassIDtoStack queue + # look for new images on the internal DetrendStackClass queue # caution with these 'if' statements: syntax errors # will make the task fail without given a good status - queuepop DetrendClassIDtoStack -var line -key 0 NEW + queuepop DetrendStackClass -var line -key 0 NEW if ("$line" == "NULL") break strpop line state - queuepush DetrendClassIDtoStack -replace -key 1:2:4 "RUN $line" + queuepush DetrendStackClass -replace -key 1:2:4 "RUN $line" - # the DetrendClassIDtoStack queue contains: + # the DetrendStackClass queue contains: # STATE DET_ID ITER TYPE CLASS_ID ## NOTE: the queue has the STATE prepended @@ -108,5 +103,5 @@ $TYPE = $word:2 $CLASS_ID = $word:3 - $CAMERA = $word:3 + $CAMERA = $word:4 stdout $CAMERA.$TYPE.$DET_ID.$ITER.$CLASS_ID.log @@ -129,15 +124,15 @@ # success task.exit 0 - queuepush DetrendClassIDtoStack -replace -key 1:2:4 "DONE $options:0" + queuepush DetrendStackClass -replace -key 1:2:4 "DONE $options:0" end # default exit status task.exit default - queuepush DetrendClassIDtoStack -replace -key 1:2:4 "FAIL $options:0" + queuepush DetrendStackClass -replace -key 1:2:4 "FAIL $options:0" end # operation times out? task.exit timeout - queuepush DetrendClassIDtoStack -replace -key 1:2:4 "TIMEOUT $options:0" + queuepush DetrendStackClass -replace -key 1:2:4 "TIMEOUT $options:0" end end Index: /trunk/ippTasks/pantasks.pro =================================================================== --- /trunk/ippTasks/pantasks.pro (revision 9103) +++ /trunk/ippTasks/pantasks.pro (revision 9104) @@ -40,7 +40,14 @@ end +macro step2.flat + exec dettool -definebyquery -det_type flat -exp_type flat -camera isp + input $scripts/detrend.process.pro + run +end + macro step2.both exec dettool -definebyquery -det_type bias -exp_type bias -camera isp exec dettool -definebyquery -det_type dark -exp_type dark -camera isp + exec dettool -definebyquery -det_type flat -exp_type flat -camera isp input $scripts/detrend.process.pro run @@ -49,4 +56,5 @@ macro step3 input $scripts/detrend.stack.pro + input $scripts/detrend.norm.pro input $scripts/detrend.resid.pro input $scripts/detrend.reject.pro Index: /trunk/ippTasks/phase0.pro =================================================================== --- /trunk/ippTasks/phase0.pro (revision 9103) +++ /trunk/ippTasks/phase0.pro (revision 9104) @@ -2,11 +2,11 @@ ## this file contains the tasks for running the phase 0 stage -## these tasks use the queue Phase0PendingImfiles -## the Phase0PendingImfiles queue contains: +## these tasks use the queue Phase0Imfiles +## the Phase0Imfiles queue contains: ## STATE EXP_KEY CLASS CLASS_ID URI-FITS ## queue keys are counted from 0 -queueinit Phase0PendingImfiles -queueinit Phase0PendingExposures +queueinit Phase0Imfiles +queueinit Phase0Exposures if ($?network == 0) $network = 1 @@ -17,5 +17,5 @@ # select images ready for phase0 analysis -# new entries are added to queue Phase0PendingImfiles +# new entries are added to queue Phase0Imfiles # compare the new list with the ones already selected task phase0.imfile.load @@ -36,29 +36,29 @@ # save stdout and stderr - # compare output with Phase0PendingImfiles queue + # compare output with Phase0Imfiles queue # only add entries which don't exist in queue queuesize stdout -var Nqueue for i 0 $Nqueue queuepop stdout -var line - queuepush Phase0PendingImfiles -uniq -key 1 "NEW $line" + queuepush Phase0Imfiles -uniq -key 1 "NEW $line" end # delete existing entries which are DONE - queuedrop Phase0PendingImfiles -key 0 DONE + queuedrop Phase0Imfiles -key 0 DONE end # locked list task.exit default - echo "*** phase0 pending imfiles: failure" - end - - # operation times out? - task.exit timeout - echo "*** phase0 pending imfiles: timeout" + echo "phase0 pending imfiles: failure" + end + + # operation times out? + task.exit timeout + echo "phase0 pending imfiles: timeout" end end # run the phase0imfile script on pending images -task phase0.imfile.process +task phase0.imfile.run periods -poll 0.5 periods -exec 1 @@ -66,5 +66,5 @@ task.exec - queuesize Phase0PendingImfiles -var N + queuesize Phase0Imfiles -var N if ($N == 0) break if ($network == 0) break @@ -73,11 +73,11 @@ # caution with these 'if' statements: syntax errors # will make the task fail without given a good status - queuepop Phase0PendingImfiles -var line -key 0 NEW + queuepop Phase0Imfiles -var line -key 0 NEW if ("$line" == "NULL") break strpop line state - queuepush Phase0PendingImfiles -replace -key 1 "RUN $line" - - ## the Phase0PendingImfiles queue contains (line has popped STATE) + queuepush Phase0Imfiles -replace -key 1 "RUN $line" + + ## the Phase0Imfiles queue contains (line has popped STATE) ## STATE EXP_KEY CLASS CLASS_ID URI-FITS list word -split $line @@ -103,5 +103,5 @@ task.exit 0 # phase0imfile.pl updates DB tables, here we just update the queue - queuepush Phase0PendingImfiles -replace -key 1 "DONE $options:0" + queuepush Phase0Imfiles -replace -key 1 "DONE $options:0" end @@ -109,5 +109,5 @@ task.exit default echo "failure for phase0imfile.pl: $options:0" - queuepush Phase0PendingImfiles -replace -key 1 "FAIL $options:0" + queuepush Phase0Imfiles -replace -key 1 "FAIL $options:0" end @@ -115,14 +115,14 @@ task.exit timeout echo "timeout for phase0imfile.pl: $options:0" - queuepush Phase0PendingImfiles -replace -key 1 "TIMEOUT $options:0" - end -end - -## these tasks use the queue Phase0PendingExposures -## the Phase0PendingExposures queue contains: + queuepush Phase0Imfiles -replace -key 1 "TIMEOUT $options:0" + end +end + +## these tasks use the queue Phase0Exposures +## the Phase0Exposures queue contains: ## STATE EXP_KEY CAMERA TELESCOPE TYPE NCLASS # select images ready for phase0 analysis -# new entries are added to queue Phase0PendingExposures +# new entries are added to queue Phase0Exposures # compare the new list with the ones already selected task phase0.exp.load @@ -141,14 +141,14 @@ local i Nqueue - # compare output with Phase0PendingExposures queue + # compare output with Phase0Exposures queue # only add entries which don't exist in queue queuesize stdout -var Nqueue for i 0 $Nqueue queuepop stdout -var line - queuepush Phase0PendingExposures -uniq -key 1 "NEW $line" + queuepush Phase0Exposures -uniq -key 1 "NEW $line" end # delete existing entries which are DONE - queuedrop Phase0PendingExposures -key 0 DONE + queuedrop Phase0Exposures -key 0 DONE end @@ -165,5 +165,5 @@ # run the phase0imfile script on pending images -task phase0.exp.process +task phase0.exp.run periods -poll 0.5 periods -exec 1.0 @@ -171,5 +171,5 @@ task.exec - queuesize Phase0PendingExposures -var N + queuesize Phase0Exposures -var N if ($N == 0) break if ($network == 0) break @@ -178,11 +178,11 @@ # caution with these 'if' statements: syntax errors # will make the task fail without given a good status - queuepop Phase0PendingExposures -var line -key 0 NEW + queuepop Phase0Exposures -var line -key 0 NEW if ("$line" == "NULL") break strpop line state - queuepush Phase0PendingExposures -replace -key 1 "RUN $line" - - ## the Phase0PendingExposures queue contains: + queuepush Phase0Exposures -replace -key 1 "RUN $line" + + ## the Phase0Exposures queue contains: ## STATE EXP_KEY CAMERA TELESCOPE TYPE NCLASS list word -split $line @@ -208,15 +208,15 @@ task.exit 0 # phase0exp.pl updates DB tables, here we just update the queue - queuepush Phase0PendingExposures -replace -key 1 "DONE $options:0" + queuepush Phase0Exposures -replace -key 1 "DONE $options:0" end # default exit status task.exit default - queuepush Phase0PendingExposures -replace -key 1 "FAIL $options:0" - end - - # operation times out? - task.exit timeout - queuepush Phase0PendingExposures -replace -key 1 "TIMEOUT $options:0" + queuepush Phase0Exposures -replace -key 1 "FAIL $options:0" + end + + # operation times out? + task.exit timeout + queuepush Phase0Exposures -replace -key 1 "TIMEOUT $options:0" end end