Index: /trunk/ippMonitor/def/p3ProcessedExp.d =================================================================== --- /trunk/ippMonitor/def/p3ProcessedExp.d (revision 9398) +++ /trunk/ippMonitor/def/p3ProcessedExp.d (revision 9398) @@ -0,0 +1,13 @@ +TABLE p3ProcessedExp +TITLE p3ProcessedExp +FILE p3ProcessedExp.php +MENU ipp.science.dat + +# field size name show link to extras +FIELD exp_tag, 15, Exposure, value +FIELD b2_uri, 8, image, image=$b2_uri, p2ProcessedImfile.php, exp_tag=$exp_tag,image=$b1_uri +FIELD bg, 8, backgnd, value +FIELD bg_mean_stdev, 8, [stdev], value +FIELD bg_stdev, 8, stdev, value +FIELD b1_uri, 20, b1_uri, none +#FIELD recipe, 15, Recipe, value Index: /trunk/ippTasks/phase2.pro =================================================================== --- /trunk/ippTasks/phase2.pro (revision 9398) +++ /trunk/ippTasks/phase2.pro (revision 9398) @@ -0,0 +1,125 @@ +## this file contains the tasks for running the phase 0 stage + +## these tasks use the queue Phase2Imfiles +## the Phase2Imfiles queue contains: +## STATE EXP_TAG CLASS CLASS_ID URI-FITS +## queue keys are counted from 0 + +queueinit Phase2Imfiles +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + +# select images ready for phase2 analysis +# new entries are added to queue Phase2Imfiles +# compare the new list with the ones already selected +task phase2.imfile.load + command p2tool -pendingimfile -simple + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + + stdout phase2.imfile.log + stderr phase2.imfile.log + + # success + task.exit 0 + local i Nqueue + + # save stdout and stderr + + # compare output with Phase2Imfiles queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + list word -split $line + $EXP_TAG = $word:0 + $CLASS_ID = $word:1 + $URI = $word:2 + queuepush Phase2Imfiles -uniq -key 1:2 "NEW $EXP_TAG $CLASS_ID $URI" + end + + # delete existing entries which are DONE + queuedrop Phase2Imfiles -key 0 DONE + end + + # locked list + task.exit default + echo "phase2 pending imfiles: failure" + end + + # operation times out? + task.exit timeout + echo "phase2 pending imfiles: timeout" + end +end + +# run the phase2imfile script on pending images +task phase2.imfile.run + periods -poll 0.5 + periods -exec 1 + periods -timeout 60 + + task.exec + queuesize Phase2Imfiles -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendImfilesToProcess queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop Phase2Imfiles -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush Phase2Imfiles -replace -key 1:2 "RUN $line" + + ## the Phase2Imfiles queue contains (line has popped STATE) + ## STATE EXP_TAG CLASS CLASS_ID URI-FITS + list word -split $line + $EXP_TAG = $word:0 + $CLASS_ID = $word:1 + $URI = $word:2 + + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end + + ## add 'p2' to the log files? + $LOG_DIR = `dirname $URI` + stdout $WORKDIR/$LOG_DIR/$EXP_TAG.$CLASS_ID.log + stderr $WORKDIR/$LOG_DIR/$EXP_TAG.$CLASS_ID.log + + # create the command line + options "$line" + echo command phase2.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI + command phase2.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI + end + + # success + task.exit 0 + # phase2imfile.pl updates DB tables, here we just update the queue + queuepush Phase2Imfiles -replace -key 1:2 "DONE $options:0" + end + + # default exit status + task.exit default + echo "failure for phase2imfile.pl: $options:0" + queuepush Phase2Imfiles -replace -key 1:2 "FAIL $options:0" + end + + # operation times out? + task.exit timeout + echo "timeout for phase2imfile.pl: $options:0" + queuepush Phase2Imfiles -replace -key 1:2 "TIMEOUT $options:0" + end +end Index: /trunk/ippTasks/phase3.pro =================================================================== --- /trunk/ippTasks/phase3.pro (revision 9398) +++ /trunk/ippTasks/phase3.pro (revision 9398) @@ -0,0 +1,118 @@ +## this file contains the tasks for running the phase 0 stage + +## these tasks use the queue Phase3Imfiles +## the Phase3Imfiles queue contains: +## STATE EXP_TAG CLASS CLASS_ID URI-FITS +## queue keys are counted from 0 + +queueinit Phase3Exposures +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + +# select images ready for phase3 analysis +# new entries are added to queue Phase3Exposures +# compare the new list with the ones already selected +task phase3.exp.load + command p3tool -pendingexp -simple + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + + stdout phase3.exp.log + stderr phase3.exp.log + + # success + task.exit 0 + local i Nqueue + + # compare output with Phase3Exposures queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + list word -split $line + $EXP_TAG = $word:0 + queuepush Phase3Exposures -uniq -key 1 "NEW $EXP_TAG" + end + + # delete existing entries which are DONE + queuedrop Phase3Exposures -key 0 DONE + end + + # default exit status + task.exit default + echo "phase3 pending exp: failure" + end + + # operation times out? + task.exit timeout + echo "phase3 pending exp: timeout" + end +end + +# run the phase3imfile script on pending images +task phase3.exp.run + periods -poll 0.5 + periods -exec 1.0 + periods -timeout 5 + + task.exec + queuesize Phase3Exposures -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendImfilesToProcess queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop Phase3Exposures -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush Phase3Exposures -replace -key 1 "RUN $line" + + ## the Phase3Exposures queue contains: + ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS + list word -split $line + $EXP_TAG = $word:0 + + # XXX add $WORKDIR/$LOG_DIR + stdout $EXP_TAG.log + stderr $EXP_TAG.log + + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end + + # create the command line + options "$line" + echo command phase3.pl --exp_tag $EXP_TAG + command phase3.pl --exp_tag $EXP_TAG + end + + # success + task.exit 0 + # phase3exp.pl updates DB tables, here we just update the queue + queuepush Phase3Exposures -replace -key 1 "DONE $options:0" + end + + # default exit status + task.exit default + queuepush Phase3Exposures -replace -key 1 "FAIL $options:0" + end + + # operation times out? + task.exit timeout + queuepush Phase3Exposures -replace -key 1 "TIMEOUT $options:0" + end +end + +## XXX add a global path to output files