Index: trunk/ippTasks/detrend.process.pro =================================================================== --- trunk/ippTasks/detrend.process.pro (revision 9032) +++ trunk/ippTasks/detrend.process.pro (revision 9081) @@ -13,4 +13,12 @@ ## STATE DET_ID TYPE EXP_KEY CLASS CLASS_ID URI +queueinit DetrendImfilesToProcess +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + # select images ready for copy # new entries are added to queue DetrendImfilesToProcess @@ -23,5 +31,7 @@ periods -exec 5 periods -timeout 30 - trange 18:00 23:59 + + stdout dettool.process.log + stderr dettool.process.log # success @@ -69,6 +79,6 @@ task dettool.raw.process periods -poll 0.5 - periods -exec 1.0 - periods -timeout 5 + periods -exec 2.0 + periods -timeout 60 task.exec @@ -90,18 +100,25 @@ ## NOTE: the queue has the STATE prepended list word -split $line - $DET_ID = $word:1 - $TYPE = $word:2 - $EXP_KEY = $word:3 - $CLASS = $word:4 - $CLASS_ID = $word:5 - $URI = $word:6 + $DET_ID = $word:0 + $TYPE = $word:1 + $EXP_KEY = $word:2 + $CLASS = $word:3 + $CLASS_ID = $word:4 + $URI = $word:5 + + stdout $EXP_KEY.detproc.$CLASS_ID.log + stderr $EXP_KEY.detproc.$CLASS_ID.log # specify choice of remote host: - host anyhost + if ($parallel) + host anyhost + else + host local + end # create example job options as a demonstration options "$line" - # ~/ipp/helpers/detrend_process.pl 1 850131b mc bias 850131b.fits 850131b - command detrend_process.pl $DET_ID $EXP_KEY $CLASS_ID $TYPE $URI $EXP_KEY + echo command detrend_process.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI + command detrend_process.pl --det_id $DET_ID --exp_id $EXP_KEY --class $CLASS --class_id $CLASS_ID --det_type bias --input_uri $URI end @@ -118,5 +135,5 @@ # operation times out? task.exit timeout - queuepush DetrendImfilesToProcess -replace -key 1:3:5 "FAIL $options:0" + queuepush DetrendImfilesToProcess -replace -key 1:3:5 "TIMEOUT $options:0" end end Index: trunk/ippTasks/detrend.reject.pro =================================================================== --- trunk/ippTasks/detrend.reject.pro (revision 9032) +++ trunk/ippTasks/detrend.reject.pro (revision 9081) @@ -13,8 +13,16 @@ ## STATE DET_ID ITER TYPE +queueinit DetrendRejectExp +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + # select images ready for copy # new entries are added to queue DetrendRejectExp # compare the new list with the ones already selected -task dettool.assess.load +task dettool.reject.load command dettool -residdetrun -simple host local @@ -22,5 +30,8 @@ periods -poll 1 periods -exec 5 - periods -timeout 30 + periods -timeout 5 + + stdout dettool.reject.log + stderr dettool.reject.log # success @@ -36,6 +47,6 @@ $DET_ID = $word:0 $ITER = $word:1 - $TYPE = $word:2 - queuepush DetrendRejectExp -uniq -key 1:2 "NEW $DET_ID $ITER $TYPE" + $DET_TYPE = $word:2 + queuepush DetrendRejectExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE" end @@ -61,8 +72,8 @@ # copy new images, sending job to desired host -task dettool.resid.process +task dettool.reject.process periods -poll 0.5 periods -exec 1.0 - periods -timeout 5 + periods -timeout 30 task.exec @@ -84,17 +95,24 @@ ## NOTE: the queue has the STATE prepended list word -split $line - $DET_ID = $word:1 - $ITER = $word:2 - $TYPE = $word:3 + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + + # I'd like to add CAMERA to the log file... + stdout isp.$DET_TYPE.$DET_ID.$ITER.log + stderr isp.$DET_TYPE.$DET_ID.$ITER.log # specify choice of remote host: - host anyhost + if ($parallel) + host anyhost + else + host local + end # create example job options as a demonstration options "$line" - # detrend_reject_exp.pl 1 0 bias - # detrend_reject_exp.pl DET_ID ITER TYPE - command detrend_reject_imfile.pl $DET_ID $ITER $TYPE - end + echo command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE + command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE + end # success @@ -110,5 +128,5 @@ # operation times out? task.exit timeout - queuepush DetrendRejectExp -replace -key 1:2 "FAIL $options:0" + queuepush DetrendRejectExp -replace -key 1:2 "TIMEOUT $options:0" end end Index: trunk/ippTasks/detrend.resid.pro =================================================================== --- trunk/ippTasks/detrend.resid.pro (revision 9032) +++ trunk/ippTasks/detrend.resid.pro (revision 9081) @@ -13,4 +13,13 @@ ## the DetrendResidImfiles queue contains: ## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN + +queueinit DetrendResidImfiles +queueinit DetrendResidExp +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end # select images ready for copy @@ -23,5 +32,8 @@ periods -poll 1 periods -exec 5 - periods -timeout 30 + periods -timeout 5 + + stdout dettool.resid.log + stderr dettool.resid.log # success @@ -38,10 +50,12 @@ list word -split $line $ITER = $word:0 - $TYPE = $word:1 + $DET_TYPE = $word:1 $DET_ID = $word:2 $EXP_KEY = $word:3 $CLASS_ID = $word:4 $URI_IN = $word:5 - queuepush DetrendResidImfiles -uniq -key 1:3:4:5 "NEW $ITER $TYPE $DET_ID $EXP_KEY $CLASS_ID $URI_IN" + ## XXX need to include the detID and iter in the key (if we have two running at the same time...) + ## Also needed to be in the output file (or path) + queuepush DetrendResidImfiles -uniq -key 1:3:4:5 "NEW $ITER $DET_TYPE $DET_ID $EXP_KEY $CLASS_ID $URI_IN" end @@ -69,6 +83,6 @@ task dettool.resid.process periods -poll 0.5 - periods -exec 1.0 - periods -timeout 5 + periods -exec 5.0 + periods -timeout 15 task.exec @@ -90,19 +104,28 @@ ## NOTE: the queue has the STATE prepended list word -split $line - $ITER = $word:1 - $TYPE = $word:2 - $DET_ID = $word:3 - $EXP_KEY = $word:4 - $CLASS_ID = $word:5 - $URI_IN = $word:6 + $ITER = $word:0 + $DET_TYPE = $word:1 + $DET_ID = $word:2 + $EXP_KEY = $word:3 + $CLASS_ID = $word:4 + $URI_IN = $word:5 + + stdout $EXP_KEY.detresid.$DET_ID.$ITER.$CLASS_ID.log + stderr $EXP_KEY.detresid.$DET_ID.$ITER.$CLASS_ID.log # specify choice of remote host: - host anyhost + if ($parallel) + host anyhost + else + host local + end # create example job options as a demonstration options "$line" # detrend_create_resid.pl DET_ID ITER EXP_KEY CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_KEY ???) - $URL_DET = $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit - command detrend_create_resid.pl $DET_ID $ITER $EXP_KEY $CLASS_ID $TYPE $URI_DET $URL_IN $EXP_KEY + $URL_DET = isp.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits + # $URL_DET = $CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits + echo command detrend_create_resid.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --class_id $CLASS_ID --det_type $DET_TYPE --detrend $URL_DET --input_uri $URI_IN + command detrend_create_resid.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --class_id $CLASS_ID --det_type $DET_TYPE --detrend $URL_DET --input_uri $URI_IN end @@ -119,5 +142,121 @@ # operation times out? task.exit timeout - queuepush DetrendResidImfiles -replace -key 1:3:4:5 "FAIL $options:0" - end -end + queuepush DetrendResidImfiles -replace -key 1:3:4:5 "TIMEOUT $options:0" + end +end + +# select images ready for copy +# new entries are added to queue DetrendResidExp +# compare the new list with the ones already selected +task dettool.residexp.load + command dettool -toresidexp -simple + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 5 + + stdout dettool.residexp.log + stderr dettool.residexp.log + + # success + task.exit 0 + # XXX is it necessary for these to be local? + # drop after this task macro is done? + local i Nqueue + + # compare output with newImage queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $EXP_KEY = $word:3 + $INCLUDE = $word:4 + queuepush DetrendResidExp -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_KEY $INCLUDE" + end + + # delete existing entries which are DONE + queuedrop DetrendResidExp -key 0 DONE + end + + # locked list + task.exit 1 + echo "new.images: exec failure" + end + + # default exit status + task.exit default + echo "new.images: unknown exit status: $EXIT" + end + + # operation times out? + task.exit timeout + echo "new.images: timeout" + end +end + +# copy new images, sending job to desired host +task dettool.residexp.process + periods -poll 0.5 + periods -exec 5.0 + periods -timeout 15 + + task.exec + queuesize DetrendResidExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendResidExp queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendResidExp -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendResidExp -replace -key 1:2:4 "RUN $line" + + ## the DetrendResidExp queue contains: + ## STATE ITER TYPE DET_ID EXP_KEY CLASS_ID URI_IN + ## NOTE: the queue has the STATE prepended + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $EXP_KEY = $word:3 + $INCLUDE = $word:4 + + stdout $EXP_KEY.detresid.$DET_ID.$ITER.log + stderr $EXP_KEY.detresid.$DET_ID.$ITER.log + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --det_type $DET_TYPE + command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_id $EXP_KEY --det_type $DET_TYPE + end + + # success + task.exit 0 + queuepush DetrendResidExp -replace -key 1:2:4 "DONE $options:0" + end + + # default exit status + task.exit default + queuepush DetrendResidExp -replace -key 1:2:4 "FAIL $options:0" + end + + # operation times out? + task.exit timeout + queuepush DetrendResidExp -replace -key 1:2:4 "TIMEOUT $options:0" + end +end Index: trunk/ippTasks/detrend.stack.pro =================================================================== --- trunk/ippTasks/detrend.stack.pro (revision 9032) +++ trunk/ippTasks/detrend.stack.pro (revision 9081) @@ -13,5 +13,13 @@ ## these tasks use the queue DetrendImfilesToProcess ## the DetrendClassIDtoStack queue contains: -## STATE DET_ID ITER TYPE CLASS_ID +## STATE DET_ID ITER TYPE CLASS_ID CAMERA + +queueinit DetrendClassIDtoStack +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end # select images ready for copy @@ -24,5 +32,8 @@ periods -poll 1 periods -exec 5 - periods -timeout 30 + periods -timeout 5 + + stdout dettool.stack.log + stderr dettool.stack.log # success @@ -42,5 +53,7 @@ $TYPE = $word:2 $CLASS_ID = $word:3 - queuepush DetrendClassIDtoStack -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID" + $CAMERA = $word:4 + echo queuepush DetrendClassIDtoStack -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA" + queuepush DetrendClassIDtoStack -uniq -key 1:2:4 "NEW $DET_ID $ITER $TYPE $CLASS_ID $CAMERA" end @@ -69,5 +82,5 @@ periods -poll 0.5 periods -exec 1.0 - periods -timeout 5 + periods -timeout 60 task.exec @@ -89,17 +102,27 @@ # STATE DET_ID ITER TYPE CLASS_ID ## NOTE: the queue has the STATE prepended + # use local variables list word -split $line - $DET_ID = $word:1 - $ITER = $word:2 - $TYPE = $word:3 - $CLASS_ID = $word:4 + $DET_ID = $word:0 + $ITER = $word:1 + $TYPE = $word:2 + $CLASS_ID = $word:3 + $CAMERA = $word:3 + + stdout $CAMERA.$TYPE.$DET_ID.$ITER.$CLASS_ID.log + stderr $CAMERA.$TYPE.$DET_ID.$ITER.$CLASS_ID.log # specify choice of remote host: - host anyhost + if ($parallel) + host anyhost + else + host local + end # create example job options as a demonstration options "$line" - # detrend_stack.pl 1 0 mc bias - command detrend_stack.pl $DET_ID $ITER $CLASS_ID $TYPE + # detrend_stack.pl --det_id 1 --iteration 0 --class_id isp --det_type bias --camera ISP || exit 1 + echo command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $TYPE --camera $CAMERA + command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $TYPE --camera $CAMERA end @@ -116,5 +139,5 @@ # operation times out? task.exit timeout - queuepush DetrendClassIDtoStack -replace -key 1:2:4 "FAIL $options:0" + queuepush DetrendClassIDtoStack -replace -key 1:2:4 "TIMEOUT $options:0" end end Index: trunk/ippTasks/pantasks.pro =================================================================== --- trunk/ippTasks/pantasks.pro (revision 9081) +++ trunk/ippTasks/pantasks.pro (revision 9081) @@ -0,0 +1,54 @@ + +$network = 1 +$parallel = 0 +$scripts = /home/kiawe/eugene/src/panstarrs/ipp.full/ippTasks + +macro init + if ($parallel) + controller host add alala + end + exec rm -f *.log + exec rm -f *.stats + exec rm -f *.detproc.*.fits + exec rm -f *.detresid.*.fits + exec rm -f *.detresid.*.jpg + exec rm -f *.detresid.*.list + exec rm -f isp.bias.*.fits + + exec upload.sh +end + +# need the ability to activate or deactivate specific tasks... + +macro step1 + init + input $scripts/phase0.pro + run +end + +# alala: dettool -tostack -simple +macro step2.bias + exec dettool -definebyquery -det_type bias -exp_type bias -camera isp + input $scripts/detrend.process.pro + run +end + +macro step2.dark + exec dettool -definebyquery -det_type dark -exp_type dark -camera isp + input $scripts/detrend.process.pro + run +end + +macro step2.both + exec dettool -definebyquery -det_type bias -exp_type bias -camera isp + exec dettool -definebyquery -det_type dark -exp_type dark -camera isp + input $scripts/detrend.process.pro + run +end + +macro step3 + input $scripts/detrend.stack.pro + input $scripts/detrend.resid.pro + input $scripts/detrend.reject.pro + run +end Index: trunk/ippTasks/phase0.pro =================================================================== --- trunk/ippTasks/phase0.pro (revision 9032) +++ trunk/ippTasks/phase0.pro (revision 9081) @@ -11,4 +11,7 @@ if ($?network == 0) $network = 1 +end +if ($?parallel == 0) + $parallel = 0 end @@ -59,6 +62,6 @@ task phase0.imfile.process periods -poll 0.5 - periods -exec 1.0 - periods -timeout 5 + periods -exec 1 + periods -timeout 60 task.exec @@ -80,17 +83,11 @@ list word -split $line - # specify choice of remote host: - # 'anyhost' specifies random remote host - # if we needed to specify a host explicitly, use a command like - # one of the following. the -required forces execution on the - # named host, and blocks the task until the host is available - # host kiawe - # host alala -required - ## XXX need to determine the appropriate host - # host anyhost - host local - echo "running phase0.imfile.process" - - echo "setting stdout, stderr: $word:1.$word:3.log" + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end + # do something better with this output target stdout $word:0.$word:2.log @@ -98,7 +95,5 @@ # create the command line - echo "setting options $line" options "$line" - echo command phase0imfile.pl --exp_id $word:0 --class_id $word:2 --uri $word:3 command phase0imfile.pl --exp_id $word:0 --class_id $word:2 --uri $word:3 @@ -113,4 +108,5 @@ # default exit status task.exit default + echo "failure for phase0imfile.pl: $options:0" queuepush Phase0PendingImfiles -replace -key 1 "FAIL $options:0" end @@ -118,5 +114,6 @@ # operation times out? task.exit timeout - queuepush Phase0PendingImfiles -replace -key 1 "FAIL $options:0" + echo "timeout for phase0imfile.pl: $options:0" + queuepush Phase0PendingImfiles -replace -key 1 "TIMEOUT $options:0" end end @@ -139,8 +136,4 @@ stdout phase0.exp.log stderr phase0.exp.log - - task.exec - break - end # success @@ -196,21 +189,18 @@ # do something better with this output target - stdout $word:1.log - stderr $word:1.log - - # specify choice of remote host: - # 'anyhost' specifies random remote host - # if we needed to specify a host explicitly, use a command like - # one of the following. the -required forces execution on the - # named host, and blocks the task until the host is available - # host kiawe - # host alala -required - ## XXX need to determine the appropriate host - host local - # host anyhost + stdout $word:0.log + stderr $word:0.log + + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end # create the command line options "$line" - command phase0exp.pl $word:1 + echo command phase0exp.pl $word:0 + command phase0exp.pl $word:0 end @@ -228,5 +218,7 @@ # operation times out? task.exit timeout - queuepush Phase0PendingExposures -replace -key 1 "FAIL $options:0" - end -end + queuepush Phase0PendingExposures -replace -key 1 "TIMEOUT $options:0" + end +end + +## XXX add a global path to output files