Index: /branches/eam_branch_00/ippTasks/.cvsignore =================================================================== --- /branches/eam_branch_00/ippTasks/.cvsignore (revision 10761) +++ /branches/eam_branch_00/ippTasks/.cvsignore (revision 10761) @@ -0,0 +1,2 @@ +.pantasks +.pcontrol Index: /branches/eam_branch_00/ippTasks/detrend.mkruns.pro =================================================================== --- /branches/eam_branch_00/ippTasks/detrend.mkruns.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/detrend.mkruns.pro (revision 10761) @@ -0,0 +1,95 @@ + +## This script defines tasks which regularly initiate the detrend +## runs. Each detrend type is given its own task, which are run once +## (or twice) per day to build the relevant detrend data + +## when one of these tasks is run, it creates a new detrend run ID in +## the database, and selects the exposures for that run on the basis +## of the specified filters + +# create a new detrend bias-creation run +task mkdetruns.megacam + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.megacam + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new MEGACAM detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end + +# create a new detrend bias-creation run +task mkdetruns.isp + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.isp + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new ISP detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end + +# create a new detrend bias-creation run +task mkdetruns.gpc + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.gpc + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new GPC detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end Index: /branches/eam_branch_00/ippTasks/detrend.norm.pro =================================================================== --- /branches/eam_branch_00/ippTasks/detrend.norm.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/detrend.norm.pro (revision 10761) @@ -0,0 +1,418 @@ + +## this file contains the tasks for running the detrend normalization stage + +## these tasks use the queue DetrendNorm +## the DetrendNorm queue contains: +## STATE DET_ID ITER + +queueinit DetrendNorm +queueinit DetrendNormStat +queueinit DetrendNormExp +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + +$DetrendNormFail = 0 +$DetrendNormStatFail = 0 +$DetrendNormExpFail = 0 + +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +macro detnorm.status + queueprint DetrendNorm + queueprint DetrendNormStat + queueprint DetrendNormExp + echo "DetrendNormFail: $DetrendNormFail" + echo "DetrendNormStatFail: $DetrendNormStatFail" + echo "DetrendNormExpFail: $DetrendNormExpFail" +end + +# select images ready for copy +# new entries are added to queue DetrendNormStat +# compare the new list with the ones already selected +task dettool.normstat.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/dettool.normstat.log + stderr $LOGSUBDIR/dettool.normstat.log + + task.exec + command dettool -tonormalizedstat -simple -limit {$DetrendNormStatFail + 20} + end + + # success + task.exit 0 + # XXX is it necessary for these to be local? + # drop after this task macro is done? + local i Nqueue + + # compare output with newImage queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $DET_ID = $word:0 + $DET_TYPE = $word:1 + $ITER = $word:2 + $CAMERA = $word:3 + queuepush DetrendNormStat -uniq -key 1:3 "NEW $DET_ID $DET_TYPE $ITER $CAMERA" + end + + # delete existing entries which are DONE + queuedrop DetrendNormStat -key 0 DONE + end + + # locked list + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task dettool.normstat.process + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + queuesize DetrendNormStat -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendNormStat queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendNormStat -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendNormStat -replace -key 1:3 "RUN $line" + + # the DetrendNormStat queue contains: + # STATE DET_ID ITER + list word -split $line + $DET_ID = $word:0 + $DET_TYPE = $word:1 + $ITER = $word:2 + $CAMERA = $word:3 + + # XXX add $WORKDIR/$LOG_DIR + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + if ($VERBOSE > 1) + echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE + end + command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE + end + + # success + task.exit 0 + queuepush DetrendNormStat -replace -key 1:3 "DONE $options:0" + end + + # default exit status + task.exit default + showcommand failure + queuepush DetrendNormStat -replace -key 1:3 "FAIL $options:0" + $DetrendNormStatFail ++ + end + + # operation times out? + task.exit timeout + showcommand timeout + queuepush DetrendNormStat -replace -key 1:3 "TIMEOUT $options:0" + $DetrendNormStatFail ++ + end +end + +# select images ready for copy +# new entries are added to queue DetrendNorm +# compare the new list with the ones already selected +task dettool.norm.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/dettool.norm.log + stderr $LOGSUBDIR/dettool.norm.log + + task.exec + command dettool -tonormalize -simple -limit {$DetrendNormFail + 20} + end + + # success + task.exit 0 + # XXX is it necessary for these to be local? + # drop after this task macro is done? + local i Nqueue + + # compare output with newImage queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $DET_TYPE = $word:0 + $CAMERA = $word:1 + $URI = $word:2 + $DET_ID = $word:3 + $ITER = $word:4 + $CLASS_ID = $word:5 + $NORM = $word:6 + queuepush DetrendNorm -uniq -key 4:5 "NEW $DET_TYPE $CAMERA $URI $DET_ID $ITER $CLASS_ID $NORM" + end + + # delete existing entries which are DONE + queuedrop DetrendNorm -key 0 DONE + end + + # locked list + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task dettool.norm.process + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + queuesize DetrendNorm -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendNorm queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendNorm -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendNorm -replace -key 4:5 "RUN $line" + + # the DetrendNorm queue contains: + # STATE DET_ID ITER + list word -split $line + $DET_TYPE = $word:0 + $CAMERA = $word:1 + $URI = $word:2 + $DET_ID = $word:3 + $ITER = $word:4 + $CLASS_ID = $word:5 + $NORM = $word:6 + + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + if ($VERBOSE > 1) + echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE + end + command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE + end + + # success + task.exit 0 + queuepush DetrendNorm -replace -key 4:5 "DONE $options:0" + end + + # default exit status + task.exit default + showcommand failure + queuepush DetrendNorm -replace -key 4:5 "FAIL $options:0" + $DetrendNormFail ++ + end + + # operation times out? + task.exit timeout + showcommand timeout + queuepush DetrendNorm -replace -key 4:5 "TIMEOUT $options:0" + $DetrendNormFail ++ + end +end + +# select images ready for copy +# new entries are added to queue DetrendNormExp +# compare the new list with the ones already selected +task dettool.normexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/dettool.normexp.log + stderr $LOGSUBDIR/dettool.normexp.log + + task.exec + command dettool -tonormalizedexp -simple -limit {$DetrendNormExpFail + 20} + end + + # success + task.exit 0 + # XXX is it necessary for these to be local? + # drop after this task macro is done? + local i Nqueue + + # compare output with newImage queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $CAMERA = $word:3 + #$TELESCOPE= $word:4 + #$EXP_TYPE = $word:5 + #$IMFILES = $word:6 + queuepush DetrendNormExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $CAMERA" + end + + # delete existing entries which are DONE + queuedrop DetrendNormExp -key 0 DONE + end + + # locked list + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task dettool.normexp.process + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + queuesize DetrendNormExp -var N + + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendNormExp queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendNormExp -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendNormExp -replace -key 1:2 "RUN $line" + + # the DetrendNormExp queue contains: + # STATE DET_ID ITER + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $CAMERA = $word:3 + #$TELESCOPE= $word:4 + #$EXP_TYPE = $word:5 + #$IMFILES = $word:6 + + # XXX add $WORKDIR/$LOG_DIR + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + if ($VERBOSE > 1) + echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE + end + command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE + end + + # success + task.exit 0 + queuepush DetrendNormExp -replace -key 1:2 "DONE $options:0" + end + + # default exit status + task.exit default + showcommand failure + queuepush DetrendNormExp -replace -key 1:2 "FAIL $options:0" + $DetrendNormExpFail ++ + end + + # operation times out? + task.exit timeout + showcommand timeout + queuepush DetrendNormExp -replace -key 1:2 "TIMEOUT $options:0" + $DetrendNormExpFail ++ + end +end Index: /branches/eam_branch_00/ippTasks/detrend.process.pro =================================================================== --- /branches/eam_branch_00/ippTasks/detrend.process.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/detrend.process.pro (revision 10761) @@ -0,0 +1,301 @@ +## this file contains the tasks for running the detrend processing stage + +# query for pending imfiles to process: +# dettool -raw -simple +# 1 bias 850131b fpa mc 850131b.fits BIAS r.MP9601 1.656000 5.496118 0.349071 0.000000 1155.934692 107.037283 46.214947 37.160000 105.480000 -133.067001 0.000000 +# this returns many columns for manual operations. we strip all but the first few: + +# process a single imfile +# detrend_process.pl 1 850131b mc bias 850131b.fits 850131b + +## these tasks use the queue DetrendProcessImfiles +## the DetrendProcessImfiles queue contains: +## STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI + +queueinit DetrendProcessImfiles +queueinit DetrendProcessExposures +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + +$DetrendProcessImfileFail = 0 +$DetrendProcessExpFail = 0 + +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +macro detproc.status + queueprint DetrendProcessImfiles + queueprint DetrendProcessExposures + echo "DetrendProcessImfileFail: $DetrendProcessImfileFail" + echo "DetrendProcessExpFail: $DetrendProcessExpFail" +end + +# select images ready for copy +# new entries are added to queue DetrendProcessImfiles +# compare the new list with the ones already selected +task dettool.process.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/detproc.imfile.log + stderr $LOGSUBDIR/detproc.imfile.log + + task.exec + command dettool -toprocessedimfile -simple -limit {$DetrendProcessImfileFail + 20} + end + + # success + task.exit 0 + # XXX is it necessary for these to be local? + # drop after this task macro is done? + local i Nqueue + + # compare output with newImage queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $DET_ID = $word:0 + $DET_TYPE = $word:1 + $EXP_TAG = $word:2 + $CLASS = $word:3 + $CLASS_ID = $word:4 + $URI = $word:5 + $CAMERA = $word:21 + queuepush DetrendProcessImfiles -uniq -key 1:3:5 "NEW $DET_ID $DET_TYPE $EXP_TAG $CLASS $CLASS_ID $URI $CAMERA" + end + + # delete existing entries which are DONE + queuedrop DetrendProcessImfiles -key 0 DONE + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand + end +end + +# copy new images, sending job to desired host +task dettool.process.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + queuesize DetrendProcessImfiles -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendProcessImfiles queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendProcessImfiles -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendProcessImfiles -replace -key 1:3:5 "RUN $line" + + ## the DetrendProcessImfiles queue contains: + # STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI + ## NOTE: the queue has the STATE prepended + list word -split $line + $DET_ID = $word:0 + $DET_TYPE = $word:1 + $EXP_TAG = $word:2 + $CLASS = $word:3 + $CLASS_ID = $word:4 + $URI = $word:5 + $CAMERA = $word:6 + + $word = `basename $EXP_TAG | tr '.' ' '` + list word -split $word + $base = $word:0 + stdout $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.$CLASS_ID.log + stderr $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.$CLASS_ID.log + exec mkdir -p $LOGSUBDIR/$base + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + if ($VERBOSE > 1) + echo command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA + end + command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA + end + + # success + task.exit 0 + queuepush DetrendProcessImfiles -replace -key 1:3:5 "DONE $options:0" + # detrend_process_imfile.pl --force_exist --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA + end + + # default exit status + task.exit default + showcommand failure + queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0" + $DetrendProcessImfileFail ++ + end + + # operation times out? + task.exit timeout + showcommand timeout + queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0" + $DetrendProcessImfileFail ++ + end +end + +# select images ready for copy +# new entries are added to queue DetrendProcessExposures +# compare the new list with the ones already selected +task dettool.processexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/detproc.exp.log + stderr $LOGSUBDIR/detproc.exp.log + + task.exec + command dettool -toprocessedexp -simple -limit {$DetrendProcessExpFail + 20} + end + + # success + task.exit 0 + # XXX is it necessary for these to be local? + # drop after this task macro is done? + local i Nqueue + + # compare output with newImage queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $EXP_TAG = $word:3 + $CAMERA = $word:4 + queuepush DetrendProcessExposures -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_TAG $CAMERA" + end + + # delete existing entries which are DONE + queuedrop DetrendProcessExposures -key 0 DONE + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task dettool.processexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + queuesize DetrendProcessExposures -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendProcessExposures queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendProcessExposures -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendProcessExposures -replace -key 1:2:4 "RUN $line" + + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $EXP_TAG = $word:3 + $CAMERA = $word:4 + + # XXX this is still a lame rule + $word = `basename $EXP_TAG | tr '.' ' '` + list word -split $word + $base = $word:0 + + stdout $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log + stderr $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log + exec mkdir -p $LOGSUBDIR/$base + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + if ($VERBOSE > 1) + echo command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA + end + command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA + end + + # success + task.exit 0 + queuepush DetrendProcessExposures -replace -key 1:2:4 "DONE $options:0" + end + + # default exit status + task.exit default + showcommand failure + queuepush DetrendProcessExposures -replace -key 1:2:4 "FAIL $options:0" + $DetrendProcessExpFail ++ + end + + # operation times out? + task.exit timeout + showcommand timeout + queuepush DetrendProcessExposures -replace -key 1:2:4 "TIMEOUT $options:0" + $DetrendProcessExpFail ++ + end +end Index: /branches/eam_branch_00/ippTasks/detrend.reject.pro =================================================================== --- /branches/eam_branch_00/ippTasks/detrend.reject.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/detrend.reject.pro (revision 10761) @@ -0,0 +1,161 @@ + +## this file contains the tasks for running the detrend processing stage + +# dettool -residdetrun -simple +# 1 0 bias +# DET_ID ITER TYPE + +# detrend_reject_exp.pl 1 0 bias +# detrend_reject_exp.pl DET_ID ITER TYPE + +## these tasks use the queue DetrendResidExp +## the DetrendRejectExp queue contains: +## STATE DET_ID ITER TYPE + +queueinit DetrendRejectExp +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + +$DetrendRejectExpFail = 0 + +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +macro detreject.status + queueprint DetrendRejectExp + echo "DetrendRejectExpFail: $DetrendRejectExpFail" +end + +# select images ready for copy +# new entries are added to queue DetrendRejectExp +# compare the new list with the ones already selected +task dettool.reject.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/dettool.reject.log + stderr $LOGSUBDIR/dettool.reject.log + + task.exec + command dettool -residdetrun -simple -limit {$DetrendRejectExpFail + 20} + end + + # success + task.exit 0 + local i Nqueue + + # compare output with newImage queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $MODE = $word:3 + $CAMERA = $word:4 + queuepush DetrendRejectExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $MODE $CAMERA" + end + + # delete existing entries which are DONE + queuedrop DetrendRejectExp -key 0 DONE + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand failure + end +end + +# copy new images, sending job to desired host +task dettool.reject.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + queuesize DetrendRejectExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendRejectExp queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendRejectExp -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendRejectExp -replace -key 1:2 "RUN $line" + + ## the DetrendRejectExp queue contains: + ## STATE DET_ID ITER TYPE + ## NOTE: the queue has the STATE prepended + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $MODE = $word:3 + $CAMERA = $word:4 + + # I'd like to add CAMERA to the log file... + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + if ($VERBOSE > 1) + echo command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE --camera $CAMERA + end + command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE --camera $CAMERA + end + + # success + task.exit 0 + queuepush DetrendRejectExp -replace -key 1:2 "DONE $options:0" + end + + # default exit status + task.exit default + showcommand failure + queuepush DetrendRejectExp -replace -key 1:2 "FAIL $options:0" + $DetrendRejectExpFail ++ + end + + # operation times out? + task.exit timeout + showcommand failure + queuepush DetrendRejectExp -replace -key 1:2 "TIMEOUT $options:0" + $DetrendRejectExpFail ++ + end +end Index: /branches/eam_branch_00/ippTasks/detrend.resid.pro =================================================================== --- /branches/eam_branch_00/ippTasks/detrend.resid.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/detrend.resid.pro (revision 10761) @@ -0,0 +1,317 @@ + +## this file contains the tasks for running the detrend processing stage + +# dettool -toresidimfile -simple +# 0 bias 1 850131b mc o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.565582 +# ITER TYPE DET_ID EXP_TAG CLASS_ID URI-IN + +# detrend_create_resid.pl 1 0 850131b mc bias bias_mc_1_0.fit o_850131b.mc.fit 850131b +# detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG) +# URL-DET: $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit + +## these tasks use the queue DetrendResidImfiles +## the DetrendResidImfiles queue contains: +## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN + +queueinit DetrendResidImfiles +queueinit DetrendResidExposures +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + +$DetrendResidImfileFail = 0 +$DetrendResidExpFail = 0 + +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +macro detresid.status + queueprint DetrendResidImfiles + queueprint DetrendResidExposures + echo "DetrendResidImfileFail: $DetrendResidImfileFail" + echo "DetrendResidExpFail: $DetrendResidExpFail" +end + +# select images ready for copy +# new entries are added to queue DetrendResidImfiles +# compare the new list with the ones already selected +task dettool.resid.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/detresid.log + stderr $LOGSUBDIR/detresid.log + + task.exec + command dettool -toresidimfile -simple -limit {$DetrendResidImfileFail + 20} + end + + # success + task.exit 0 + # XXX is it necessary for these to be local? + # drop after this task macro is done? + local i Nqueue + + # compare output with newImage queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $MODE = $word:3 + $EXP_TAG = $word:4 + $CLASS_ID = $word:5 + $URI = $word:6 + $DET_URI = $word:7 + $CAMERA = $word:8 + ## must include the detID and iter in the key (if we have two running at the same time...) + ## Also needed to be in the output file (or path) + queuepush DetrendResidImfiles -uniq -key 1:2:5:6 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $CLASS_ID $URI $DET_URI $CAMERA" + end + + # delete existing entries which are DONE + queuedrop DetrendResidImfiles -key 0 DONE + end + + # locked list + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task dettool.resid.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + queuesize DetrendResidImfiles -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendResidImfiles queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendResidImfiles -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendResidImfiles -replace -key 1:2:5:6 "RUN $line" + + ## the DetrendResidImfiles queue contains: + ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN + ## NOTE: the queue has the STATE prepended + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $MODE = $word:3 + $EXP_TAG = $word:4 + $CLASS_ID = $word:5 + $URI = $word:6 + $DET_URI = $word:7 + $CAMERA = $word:8 + + $word = `basename $EXP_TAG | tr '.' ' '` + list word -split $word + $base = $word:0 + stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log + stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log + exec mkdir -p $LOGSUBDIR/$base + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG ???) + # need to parse the subdir path for the detrend URL + # $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits + echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE + command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE + end + + # success + task.exit 0 + queuepush DetrendResidImfiles -replace -key 1:2:5:6 "DONE $options:0" + end + + # default exit status + task.exit default + showcommand failure + queuepush DetrendResidImfiles -replace -key 1:2:5:6 "FAIL $options:0" + $DetrendResidImfileFail ++ + end + + # operation times out? + task.exit timeout + showcommand timeout + queuepush DetrendResidImfiles -replace -key 1:2:6:6 "TIMEOUT $options:0" + $DetrendResidImfileFail ++ + end +end + +# select images ready for copy +# new entries are added to queue DetrendResidExposures +# compare the new list with the ones already selected +task dettool.residexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/detresid.exp.log + stderr $LOGSUBDIR/detresid.exp.log + + task.exec + command dettool -toresidexp -simple -limit {$DetrendResidExpFail + 20} + end + + # success + task.exit 0 + # XXX is it necessary for these to be local? + # drop after this task macro is done? + local i Nqueue + + # compare output with newImage queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $MODE = $word:3 + $EXP_TAG = $word:4 + $INCLUDE = $word:5 + $CAMERA = $word:6 + queuepush DetrendResidExposures -uniq -key 1:2:5 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $INCLUDE $CAMERA" + end + + # delete existing entries which are DONE + queuedrop DetrendResidExposures -key 0 DONE + end + + # locked list + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task dettool.residexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + queuesize DetrendResidExposures -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendResidExposures queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendResidExposures -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendResidExposures -replace -key 1:2:5 "RUN $line" + + ## the DetrendResidExposures queue contains: + ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN + ## NOTE: the queue has the STATE prepended + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $MODE = $word:3 + $EXP_TAG = $word:4 + $INCLUDE = $word:5 + $CAMERA = $word:6 + + # XXX this is still a lame rule + $word = `basename $EXP_TAG | tr '.' ' '` + list word -split $word + $base = $word:0 + + stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log + stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log + exec mkdir -p $LOGSUBDIR/$base + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + if ($VERBOSE > 1) + echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA + end + command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA + end + + # success + task.exit 0 + queuepush DetrendResidExposures -replace -key 1:2:5 "DONE $options:0" + end + + # default exit status + task.exit default + showcommand failure + queuepush DetrendResidExposures -replace -key 1:2:5 "FAIL $options:0" + $DetrendResidExpFail ++ + end + + # operation times out? + task.exit timeout + showcommand timeout + queuepush DetrendResidExposures -replace -key 1:2:5 "TIMEOUT $options:0" + $DetrendResidExpFail ++ + end +end Index: /branches/eam_branch_00/ippTasks/detrend.stack.pro =================================================================== --- /branches/eam_branch_00/ippTasks/detrend.stack.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/detrend.stack.pro (revision 10761) @@ -0,0 +1,162 @@ +## this file contains the tasks for running the detrend stacking stage + +# get a list of imfiles to stack +# dettool -tostack -simple +# 1 0 bias mc +# DET_ID ITER TYPE CLASS_ID + +# function to stack the imfiles: +# detrend_stack.pl 1 0 mc bias + +## these tasks use the queue DetrendStackClass +## the DetrendStackClass queue contains: +## STATE DET_ID ITER TYPE CLASS_ID CAMERA + +queueinit DetrendStackClass +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + +$DetrendStackClassFail = 0 + +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +macro detstack.status + queueprint DetrendStackClass + echo "DetrendStackClassFail: $DetrendStackClassFail" +end + +# select images ready for copy +# new entries are added to queue DetrendStackClass +# compare the new list with the ones already selected +task dettool.stack.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/detstack.log + stderr $LOGSUBDIR/detstack.log + + task.exec + command dettool -tostacked -simple -limit {$DetrendStackClassFail + 20} + end + + # success + task.exit 0 + local i Nqueue + + # compare output with newImage queue + # only add entries which do not exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $CLASS_ID = $word:3 + $CAMERA = $word:4 + queuepush DetrendStackClass -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $CLASS_ID $CAMERA" + end + + # delete existing entries which are DONE + queuedrop DetrendStackClass -key 0 DONE + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task dettool.stack.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 120 + + task.exec + queuesize DetrendStackClass -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendStackClass queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop DetrendStackClass -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush DetrendStackClass -replace -key 1:2:4 "RUN $line" + + # the DetrendStackClass queue contains: + # STATE DET_ID ITER TYPE CLASS_ID + ## NOTE: the queue has the STATE prepended + # use local variables + list word -split $line + $DET_ID = $word:0 + $ITER = $word:1 + $DET_TYPE = $word:2 + $CLASS_ID = $word:3 + $CAMERA = $word:4 + + # XXX add $WORKDIR/$LOG_DIR + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # create example job options as a demonstration + options "$line" + if ($VERBOSE > 1) + echo command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA + end + command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA + end + + # success + task.exit 0 + queuepush DetrendStackClass -replace -key 1:2:4 "DONE $options:0" + end + + # default exit status + task.exit default + showcommand failure + queuepush DetrendStackClass -replace -key 1:2:4 "FAIL $options:0" + $DetrendStackClassFail ++ + end + + # operation times out? + task.exit timeout + showcommand timeout + queuepush DetrendStackClass -replace -key 1:2:4 "TIMEOUT $options:0" + $DetrendStackClassFail ++ + end +end Index: /branches/eam_branch_00/ippTasks/mkdetrend.pro =================================================================== --- /branches/eam_branch_00/ippTasks/mkdetrend.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/mkdetrend.pro (revision 10761) @@ -0,0 +1,8 @@ + +input detrend.mkruns.pro +input detrend.process.pro +input detrend.stack.pro +input detrend.norm.pro +input detrend.resid.pro +input detrend.assess.pro + Index: /branches/eam_branch_00/ippTasks/notes.txt =================================================================== --- /branches/eam_branch_00/ippTasks/notes.txt (revision 10761) +++ /branches/eam_branch_00/ippTasks/notes.txt (revision 10761) @@ -0,0 +1,170 @@ + +2006.08.28 + +- ipptools need to use expID:camera:telescope as unique key (ie, not expID alone) +- p0tool -pendingimfile -simple needs to report the expKey (ie, not expID) +- + + +This directory contains the PanTasks scripts which define the steps +performed by the IPP. These scripts use the ippTools to decide what +images need to be processed, and to update the Metadata Database with +the current state of the analysis. The analysis is performed using +IPP programs which are distributed to the cluster computers as needed. + +1. The Detrend Creation Scripts + +The detrend creation process is among the more complex elements of the +IPP. There are N major steps which must be performed: + +1.1. Define New Detrend Runs + +The script 'detrend.mkruns.pro' defines tasks which initiate new +Detrend Runs. There is a separate task for each camera automatically +processed by the system (mkdetruns.CAMERA). These tasks are executed +on a regular, but infrequent basis. The tasks call external helper +programs which actually run the command dettools -define. The helper +programs are thus responsible for identifying all of the types of +detrend images to be run for a given camera, as well as the correct +set of filters to apply. These tasks are run locally and only affect +the database. PanTasks does not track the detrend runs which are +created. + +- we may need to define these within the PanTasks scripts +- how frequently are these tasks run (1/day?) +- if no raw images are selected by the query, no new detrun should be + created. +- we may need different schedules for the dark (bias/dark) and the + light (flat/fringe) data. And yet another schedule for sky data + (skyflat/skyfringe). + +1.2. Process new raw imfiles + +The script 'detrend.process.pro' defines tasks which processes the new +raw imfiles for any detrend run. This script defines two tasks, +'dettool.raw.load' and 'dettool.raw.process'. The first script +occasionally polls the Metadata Database to select raw imfiles which +have been identified as part of a detrend run. The identified raw +imfiles are added to an internal queue, DetrendImfilesToProcess. Any +raw imfile already in the list is ignored. The imfiles are added to +the queue with a state of NEW. The second task examines the queue for +imfiles with state NEW and defines a ppImage analysis job for that +imfile. The job is sent to pcontrol for processing on the distributed +cluster. At this point, the corresponding queue entry is set to the +state RUN. Upon success, the queue entry is set to DONE and the +database is notified of the successful operation. The DONE entries +are occasionally pruned from the queue; this must be done in the same +task as the initial query and the update of the list based on the +database results. + +- we need to define the columns in the output of dettool -raw +- we need a method to associate an imfile with a node in the cluster +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs +- we need a method to identify the correct recipe (function of type?) + +1.3. Stack new processed imfiles + +The script 'detrend.stack.pro' defines tasks which stack the newly +processed imfiles for any detrend run iteration. This script defines +two tasks, 'dettool.stack.load and 'dettool.stack.process. The first +script occasionally polls the Metadata Database to select class IDs +for detrend runs for which the input imfiles have all been processed. +The identified detRun/classID combination is added to an internal +queue, DetrendClassIDtoStack. The entries are added to the queue with +a state of NEW. The second task examines the queue for entries with a +state of NEW. It defines a ppMerge analysis job for that +detID/classID. The helper script selects the corresponding input +imfiles and passes them to ppMerge. The job is sent to pcontrol for +processing on the distributed cluster. At this point, the +corresponding queue entry is set to the state RUN. Upon success, the +queue entry is set to DONE and the database is notified of the +successful operation. The DONE entries are occasionally pruned from +the queue; this must be done in the same task as the initial query and +the update of the list based on the database results. + +- we need to define the columns in the output of dettool -chip -processed +- we need a method to associate a classID with a node in the cluster +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) + + +1.4. Normalize stacked masters + +The script 'detrend.norm.pro' defines tasks which normalize the newly +processed stacks for any detrend run iteration. This script defines +two tasks, 'dettool.norm.load and 'dettool.norm.process. The first +script occasionally polls the Metadata Database to select detrun IDs +for which the classes have all been stacked. The identified +detRunID/iteration combination is added to an internal queue, +DetrendRunToNormalize. The entries are added to the queue with a +state of NEW. The second task examines the queue for entries with a +state of NEW. It defines a ppNorm analysis job for that +detID/iteration. The job is sent to pcontrol for processing on the +distributed cluster. At this point, the corresponding queue entry is +set to the state RUN. Upon success, the queue entry is set to DONE and +the database is notified of the successful operation. The DONE +entries are occasionally pruned from the queue; this must be done in +the same task as the initial query and the update of the list based on +the database results. + +- we need to define the columns in the output of dettool +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) +- we need to define the ppNorm operation inputs and outputs better + + +1.5. Generate Resid Imfiles + +The script 'detrend.resid.pro' defines tasks which apply the newly +generated detrend image to the corresponding input images to generate +the residual images. This script defines two tasks, +'dettool.resid.load and 'dettool.resid.process. The first script +occasionally polls the Metadata Database to select imfiles for which +the master detrend images has been normalized. The identified imfiles +are added to an internal queue, DetrendResidImfiles. The entries are +added to the queue with a state of NEW. The second task examines the +queue for entries with a state of NEW. It defines a ppImage analysis +job for that imfile. The job is sent to pcontrol for processing on +the distributed cluster. At this point, the corresponding queue entry +is set to the state RUN. Upon success, the queue entry is set to DONE +and the database is notified of the successful operation. The DONE +entries are occasionally pruned from the queue; this must be done in +the same task as the initial query and the update of the list based on +the database results. + +- we need to define the columns in the output of dettool +- can we select the imfiles here without apriori knowledge that the + master has been normalized? +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) +- we need a method to pass the matching detrend image to ppImage in + the correct context (ie, a bias needs to be supplied to the -bias element...) + +1.6. Assess Detrend Results + +The script 'detrend.assess.pro' defines tasks which assess the +residual statistics for completed detrend runs. This script defines +two tasks, 'dettool.assess.load and 'dettool.assess.process. The +first script occasionally polls the Metadata Database to select +detrunIDs/iterations for which the resid imfiles have all been +generated. The identified detruns are added to an internal queue, +DetrendRunToAssess. The entries are added to the queue with a state +of NEW. The second task examines the queue for entries with a state +of NEW. It defines an analysis job for that collection. The job is +sent to pcontrol for processing on the distributed cluster. At this +point, the corresponding queue entry is set to the state RUN. Upon +success, the queue entry is set to DONE and the database is notified +of the successful operation. The DONE entries are occasionally pruned +from the queue; this must be done in the same task as the initial +query and the update of the list based on the database results. + +- the assess tool needs to trigger a new run, if needed. +- what is the command to run on the resid statistics? +- we need to define the columns in the output of dettool +- can we select the detrend runs here? +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) Index: /branches/eam_branch_00/ippTasks/p0tool.sh =================================================================== --- /branches/eam_branch_00/ippTasks/p0tool.sh (revision 10761) +++ /branches/eam_branch_00/ippTasks/p0tool.sh (revision 10761) @@ -0,0 +1,7 @@ +#!/bin/csh -f +if ($#argv != 1) then + echo "USAGE: p0tool.sh (Nselect)" + exit 2 +endif + +p0tool -pendingimfile -simple | head -n $1 Index: /branches/eam_branch_00/ippTasks/pantasks.pro =================================================================== --- /branches/eam_branch_00/ippTasks/pantasks.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/pantasks.pro (revision 10761) @@ -0,0 +1,81 @@ + +# XXX make these upper-case +$network = 1 +$parallel = 1 +$VERBOSE = 1 + +# $scripts = $HOME/src/pantasks/ipp/ippTasks +$scripts = . +# $LOGDIR = `ipp_datapath.pl path://ESSENCE` +# $LOGDIR = `ipp_datapath.pl path://SIMTEST` +# $LOGDIR = `ipp_datapath.pl path://ISPTEST` +# $LOGDIR = $LOGDIR/logs +# exec mkdir -p $LOGDIR + +$loadpoll = 1 +$loadexec = 5 +$runpoll = 0.5 +$runexec = 1 + +macro init.essence + $LOGDIR = `ipp_datapath.pl path://ESSENCE` + $LOGDIR = $LOGDIR/logs + exec mkdir -p $LOGDIR + + if ($parallel) + controller exit true + controller host add sn2 + controller host add sn3 + controller host add sn4 + controller host add sn5 + end +end + +macro init.isptest + $LOGDIR = `ipp_datapath.pl path://ISPTEST` + $LOGDIR = $LOGDIR/logs + exec mkdir -p $LOGDIR + + if ($parallel) + controller exit true + controller host add po02 + controller host add po03 + controller host add po04 + controller host add po05 + controller host add po06 + controller host add po07 + controller host add po08 + controller host add po09 + controller host add po10 + end +end + +# need the ability to activate or deactivate specific tasks... + +macro load.tasks + input $scripts/phase0.pro + input $scripts/phase2.pro +# input $scripts/phase3.pro + + input $scripts/detrend.process.pro + input $scripts/detrend.stack.pro + input $scripts/detrend.norm.pro + input $scripts/detrend.resid.pro + input $scripts/detrend.reject.pro +end + +macro showcommand + local n + + if ($VERBOSE) + $command = $taskarg:0 + for n 1 $taskarg:n + $command = $command $taskarg:$n + end + if ($0 == 2) + echo "$1 for: $command" + else + echo "command: $command" + end + end +end Index: /branches/eam_branch_00/ippTasks/phase0.pro =================================================================== --- /branches/eam_branch_00/ippTasks/phase0.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/phase0.pro (revision 10761) @@ -0,0 +1,296 @@ +## this file contains the tasks for running the phase 0 stage + +## these tasks use the queue Phase0Imfiles +## the Phase0Imfiles queue contains: +## STATE EXP_TAG CLASS CLASS_ID URI-FITS +## queue keys are counted from 0 + +queueinit Phase0Imfiles +queueinit Phase0Exposures +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + +$Phase0ImfileFail = 0 +$Phase0ExpFail = 0 + +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end +$LOGSUBDIR = $LOGDIR/phase0 +exec mkdir -p $LOGSUBDIR + +macro phase0.status + queueprint Phase0Imfiles + queueprint Phase0Exposures + echo "Phase0ImfileFail: $Phase0ImfileFail" + echo "Phase0ExpFail: $Phase0ExpFail" +end + +# select images ready for phase0 analysis +# new entries are added to queue Phase0Imfiles +# compare the new list with the ones already selected +task phase0.imfile.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/phase0.imfile.log + stderr $LOGSUBDIR/phase0.imfile.log + + task.exec + command p0tool -pendingimfile -simple -limit {$Phase0ImfileFail + 20} + end + + # success + task.exit 0 + local i Nqueue + + # compare output with Phase0Imfiles queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $EXP_TAG = $word:0 + $CLASS = $word:1 + $CLASS_ID = $word:2 + $URI = $word:3 + queuepush Phase0Imfiles -uniq -key 1 "NEW $EXP_TAG $CLASS $CLASS_ID $URI" + end + + # delete existing entries which are DONE + queuedrop Phase0Imfiles -key 0 DONE + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + +# run the phase0imfile script on pending images +task phase0.imfile.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + queuesize Phase0Imfiles -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendImfilesToProcess queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop Phase0Imfiles -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush Phase0Imfiles -replace -key 1 "RUN $line" + + ## the Phase0Imfiles queue contains (line has popped STATE) + ## STATE EXP_TAG CLASS CLASS_ID URI-FITS + list word -split $line + $EXP_TAG = $word:0 + $CLASS = $word:1 + $CLASS_ID = $word:2 + $URI = $word:3 + + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end + + # XXX this is still a lame rule + $word = `basename $EXP_TAG | tr '.' ' '` + list word -split $word + $base = $word:0 + stdout $LOGSUBDIR/$base/$EXP_TAG.p0.log + stderr $LOGSUBDIR/$base/$EXP_TAG.p0.log + exec mkdir -p $LOGSUBDIR/$base + + # create the command line + options "$line" + if ($VERBOSE > 1) + echo command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI + end + command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI + end + + # success + task.exit 0 + # phase0imfile.pl updates DB tables, here we just update the queue + queuepush Phase0Imfiles -replace -key 1 "DONE $options:0" + end + + # default exit status + task.exit default + showcommand failure + queuepush Phase0Imfiles -replace -key 1 "FAIL $options:0" + $Phase0ImfileFail ++ + end + + # operation times out? + task.exit timeout + showcommand timeout + queuepush Phase0Imfiles -replace -key 1 "TIMEOUT $options:0" + $Phase0ImfileFail ++ + end +end + + + +## these tasks use the queue Phase0Exposures +## the Phase0Exposures queue contains: +## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS + +# select images ready for phase0 analysis +# new entries are added to queue Phase0Exposures +# compare the new list with the ones already selected +task phase0.exp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/phase0.exp.log + stderr $LOGSUBDIR/phase0.exp.log + + task.exec + command p0tool -pendingexp -simple -limit {$Phase0ExpFail + 20} + end + + # success + task.exit 0 + local i Nqueue + + # compare output with Phase0Exposures queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $EXP_TAG = $word:0 + # $EXP_ID = $word:1 + # $CAMERA = $word:2 + # $TELESCOPE = $word:3 + # $DATEOBS = $word:4 + # $TYPE = $word:5 + # $IMFILES = $word:6 + queuepush Phase0Exposures -uniq -key 1 "NEW $EXP_TAG" + end + + # delete existing entries which are DONE + queuedrop Phase0Exposures -key 0 DONE + end + + # default exit status + task.exit default + showcommand + end + + # operation times out? + task.exit timeout + showcommand + end +end + + + +# run the phase0imfile script on pending images +task phase0.exp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + queuesize Phase0Exposures -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendImfilesToProcess queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop Phase0Exposures -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush Phase0Exposures -replace -key 1 "RUN $line" + + ## the Phase0Exposures queue contains: + ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS + list word -split $line + $EXP_TAG = $word:0 + + # XXX this is still a lame rule + $word = `basename $EXP_TAG | tr '.' ' '` + list word -split $word + $base = $word:0 + stdout $LOGSUBDIR/$base/$EXP_TAG.p0.log + stderr $LOGSUBDIR/$base/$EXP_TAG.p0.log + exec mkdir -p $LOGSUBDIR/$base + + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end + + # create the command line + options "$line" + if ($VERBOSE > 1) + echo command phase0_exp.pl --exp_tag $EXP_TAG + end + command phase0_exp.pl --exp_tag $EXP_TAG + end + + # success + task.exit 0 + # phase0exp.pl updates DB tables, here we just update the queue + queuepush Phase0Exposures -replace -key 1 "DONE $options:0" + end + + # default exit status + task.exit default + showcommand + queuepush Phase0Exposures -replace -key 1 "FAIL $options:0" + $Phase0ExpFail + end + + # operation times out? + task.exit timeout + showcommand + queuepush Phase0Exposures -replace -key 1 "TIMEOUT $options:0" + $Phase0ExpFail + end +end + +## XXX add a global path to output files Index: /branches/eam_branch_00/ippTasks/phase2.pro =================================================================== --- /branches/eam_branch_00/ippTasks/phase2.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/phase2.pro (revision 10761) @@ -0,0 +1,159 @@ +## this file contains the tasks for running the phase 0 stage + +## these tasks use the queue Phase2Imfiles +## the Phase2Imfiles queue contains: +## STATE EXP_TAG CLASS CLASS_ID URI-FITS +## queue keys are counted from 0 + +queueinit Phase2Imfiles +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + +$Phase2ImfileFail = 0 + +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end +$LOGSUBDIR = $LOGDIR/phase2 +exec mkdir -p $LOGSUBDIR + +macro phase2.status + queueprint Phase2Imfiles + echo "Phase2ImfileFail: $Phase2ImfileFail" +end + +# select images ready for phase2 analysis +# new entries are added to queue Phase2Imfiles +# compare the new list with the ones already selected +task phase2.imfile.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/phase2.imfile.log + stderr $LOGSUBDIR/phase2.imfile.log + + task.exec + # command p2tool -pendingimfile -simple -limit {$Phase2ImfileFail + 20} + command p2tool -pendingimfile -simple + end + + # success + task.exit 0 + local i Nqueue + + # save stdout and stderr + + # compare output with Phase2Imfiles queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $EXP_TAG = $word:0 + $CLASS_ID = $word:1 + $URI = $word:2 + queuepush Phase2Imfiles -uniq -key 1:2 "NEW $EXP_TAG $CLASS_ID $URI" + end + + # delete existing entries which are DONE + queuedrop Phase2Imfiles -key 0 DONE + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the phase2imfile script on pending images +task phase2.imfile.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 60 + + task.exec + queuesize Phase2Imfiles -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendImfilesToProcess queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop Phase2Imfiles -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush Phase2Imfiles -replace -key 1:2 "RUN $line" + + ## the Phase2Imfiles queue contains (line has popped STATE) + ## STATE EXP_TAG CLASS CLASS_ID URI-FITS + list word -split $line + $EXP_TAG = $word:0 + $CLASS_ID = $word:1 + $URI = $word:2 + + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end + + ## add 'p2' to the log files? + # XXX this is still a lame rule + $word = `basename $EXP_TAG | tr '.' ' '` + list word -split $word + $base = $word:0 + stdout $LOGSUBDIR/$base/$EXP_TAG.$CLASS_ID.log + stderr $LOGSUBDIR/$base/$EXP_TAG.$CLASS_ID.log + exec mkdir -p $LOGSUBDIR/$base + + # create the command line + options "$line" + if ($VERBOSE > 1) + echo command phase2.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI + end + ## XXX this is annoying: phase2.pl now requires camera :: it should not need this, it has the file. + command phase2.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --camera CTIO_MOSAIC2 + end + + # success + task.exit 0 + # phase2imfile.pl updates DB tables, here we just update the queue + queuepush Phase2Imfiles -replace -key 1:2 "DONE $options:0" + end + + # default exit status + task.exit default + showcommand failure + queuepush Phase2Imfiles -replace -key 1:2 "FAIL $options:0" + $Phase2ImfileFail ++ + end + + # operation times out? + task.exit timeout + showcommand timeout + queuepush Phase2Imfiles -replace -key 1:2 "TIMEOUT $options:0" + $Phase2ImfileFail ++ + end +end Index: /branches/eam_branch_00/ippTasks/phase3.pro =================================================================== --- /branches/eam_branch_00/ippTasks/phase3.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/phase3.pro (revision 10761) @@ -0,0 +1,152 @@ +## this file contains the tasks for running the phase 0 stage + +## these tasks use the queue Phase3Exposures +## the Phase3Exposures queue contains: +## STATE EXP_TAG CLASS CLASS_ID URI-FITS +## queue keys are counted from 0 + +queueinit Phase3Exposures +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + +$Phase3ExposureFail = 0 + +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end +$LOGSUBDIR = $LOGDIR/phase3 +exec mkdir -p $LOGSUBDIR + +macro phase3.status + queueprint Phase3Exposures + echo "Phase3ExposureFail: $Phase3ExposureFail" +end + +# select images ready for phase3 analysis +# new entries are added to queue Phase3Exposures +# compare the new list with the ones already selected +task phase3.exp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/phase3.exp.log + stderr $LOGSUBDIR/phase3.exp.log + + task.exec + command p3tool -pendingexp -simple + end + + # success + task.exit 0 + local i Nqueue + + # compare output with Phase3Exposures queue + # only add entries which don't exist in queue + queuesize stdout -var Nqueue + for i 0 $Nqueue + queuepop stdout -var line + if ($VERBOSE > 2) + echo $line + end + list word -split $line + $EXP_TAG = $word:0 + queuepush Phase3Exposures -uniq -key 1 "NEW $EXP_TAG" + end + + # delete existing entries which are DONE + queuedrop Phase3Exposures -key 0 DONE + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the phase3exposure script on pending images +task phase3.exp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 5 + + task.exec + queuesize Phase3Exposures -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images on the internal DetrendExposuresToProcess queue + # caution with these 'if' statements: syntax errors + # will make the task fail without given a good status + queuepop Phase3Exposures -var line -key 0 NEW + if ("$line" == "NULL") break + + strpop line state + queuepush Phase3Exposures -replace -key 1 "RUN $line" + + ## the Phase3Exposures queue contains: + ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS + list word -split $line + $EXP_TAG = $word:0 + + # XXX this is still a lame rule + $word = `basename $EXP_TAG | tr '.' ' '` + list word -split $word + $base = $word:0 + stdout $LOGSUBDIR/$base/$EXP_TAG.log + stderr $LOGSUBDIR/$base/$EXP_TAG.log + exec mkdir -p $LOGSUBDIR/$base + + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end + + # create the command line + options "$line" + if ($VERBOSE > 1) + echo command phase3.pl --exp_tag $EXP_TAG + end + command phase3.pl --exp_tag $EXP_TAG --camera CTIO_MOSAIC2 + end + + # success + task.exit 0 + # phase3exp.pl updates DB tables, here we just update the queue + queuepush Phase3Exposures -replace -key 1 "DONE $options:0" + end + + # default exit status + task.exit default + showcommand failure + queuepush Phase3Exposures -replace -key 1 "FAIL $options:0" + $Phase3ExposureFail ++ + end + + # operation times out? + task.exit timeout + showcommand timeout + queuepush Phase3Exposures -replace -key 1 "TIMEOUT $options:0" + $Phase3ExposureFail ++ + end +end + +## XXX add a global path to output files Index: /branches/eam_branch_00/ippTasks/summit.copy.pro =================================================================== --- /branches/eam_branch_00/ippTasks/summit.copy.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/summit.copy.pro (revision 10761) @@ -0,0 +1,308 @@ +## PanTasks scripts for Summit Copy + +$uri = http://otis/ds/skyprobe/ +$camera = skyprobe +$telescope = ps1 +$workdir = /home/moanui/jhoblitt/workdir + +if ($?parallel == 0) + $parallel = 0 +end + +# queue for summit exps that need to be queried +queueinit pzPendingExp +# queue for summit imfiles that need to be downloaded +queueinit pzPendingImfile +# queue for downloaded files that need to registered with pztool -copydone +queueinit pzCopyDone + +# run pzgetexp periodically to populate pzPendingExp +task pzgetexp + command pzgetexp -uri $uri -inst $camera -telescope $telescope + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 30 + periods -poll 10 + periods -timeout 30 + + task.exit 0 + end + + task.exit default + echo "phaseZ pzgetexp: failure" + end + task.exit timeout + echo "phaseZ pzgetexp: timeout" + end +end + +# build a queue of exps/filesetids that need to be queried +task pztool.pendingexp + command pztool -pendingexp -simple + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 30 + periods -poll 10 + periods -timeout 30 + + # success + task.exit 0 + local i Nstdout + queuesize stdout -var Nstdout + for i 0 $Nstdout + queuepop stdout -var line +# list word -split $line +# $exp_id = $word:0 +# $camera = $word:1 +# $telescope = $word:2 +# $dateobs = $word:3 +# $exp_type = $word:4 +# $uri = $word:5 +# $imfiles = $word:6 +# queuepush pzPendingExp -uniq -key 1:2:3 "NEW $uri $exp_id $camera $telescope $dateobs $exp_type $uri $imfiles" + queuepush pzPendingExp -uniq -key 1:2:3 "NEW $line" + end + + queuedrop pzPendingExp -key 0 DONE + end + + task.exit default + echo "phaseZ pending exp: failure" + end + task.exit timeout + echo "phaseZ pending exp: timeout" + end +end + +# run pzgetimfiles on pending exps +task pzgetimfile + periods -exec 5 + periods -poll 5 + periods -timeout 15 + + task.exec + queuesize pzPendingExp -var N + if ($N == 0) break + + # get a line out of the queue + queuepop pzPendingExp -var line -key 0 NEW + + # take the state out of the line + strpop line state + + # put the line back into the queue with a stea of RUN + queuepush pzPendingExp -replace -key 1:2:3 "RUN $line" + + # the STATE has already been removed from the line + list word -split $line + $exp_id = $word:0 + $camera = $word:1 + $telescope = $word:2 + $dateobs = $word:3 + $exp_type = $word:4 + $uri = $word:5 + $imfiles = $word:6 + + # specify choice of remote host:(need to choose based on chips) +# if ($parallel) +# host anyhost +# else +# host local +# end + + # store the current line + options "$line" + + echo command pzgetimfiles -uri $uri -filesetid $exp_id -inst $camera -telescope $telescope + command pzgetimfiles -uri $uri -filesetid $exp_id -inst $camera -telescope $telescope + end + + # success + task.exit 0 + queuepush pzPendingExp -replace -key 1:2:3 "DONE $options:0" + end + + task.exit default + queuepush pzPendingExp -replace -key 1:2:3 "FAIL $options:0" + end + + task.exit timeout + queuepush pzPendingExp -replace -key 1:2:3 "TIMEOUT $options:0" + end +end + +# build a queue of imfiles/files that need to be downloaded +task pztool.pendingimfile + command pztool -pendingimfile -simple + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 30 + periods -poll 10 + periods -timeout 30 + + # success + task.exit 0 + local i Nstdout + queuesize stdout -var Nstdout + for i 0 $Nstdout + queuepop stdout -var line +# list word -split $line +# $exp_id = $word:0 +# $camera = $word:1 +# $telescope = $word:2 +# $file_id = $word:3 +# $bytes = $word:4 +# $md5sum = $word:5 +# $class = $word:6 +# $class_id = $word:7 +# $uri = $word:8 +# $exp_tag = $word:9 + + queuepush pzPendingImfile -uniq -key 1:2:3:7:8 "NEW $line" + end + + # delete existing entries which are DONE + queuedrop pzPendingImfile -key 0 DONE + end + + task.exit default + echo "phaseZ pending exp: failure" + end + task.exit timeout + echo "phaseZ pending exp: timeout" + end +end + +# get dsget to retreive an imfile and then call pztool -copydone +task dsget + periods -exec 5 + periods -poll 5 + periods -timeout 30 + + task.exec + queuesize pzPendingImfile -var N + if ($N == 0) break + + # get a line out of the queue + queuepop pzPendingImfile -var line -key 0 NEW + + # take the state out of the line + strpop line state + + # put the line back into the queue with a stea of RUN + queuepush pzPendingImfile -replace -key 1:2:3:7:8 "RUN $line" + + # the STATE has already been removed from the line + list word -split $line + $exp_id = $word:0 + $camera = $word:1 + $telescope = $word:2 + $file_id = $word:3 + $bytes = $word:4 + $md5sum = $word:5 + $class = $word:6 + $class_id = $word:7 + $uri = $word:8 + $exp_tag = $word:9 + +# # specify choice of remote host:(need to choose based on chips) +# if ($parallel) +# host anyhost +# else +# host local +# end + + # store the current line without the state + options "$line" + + echo "parsing line $line" + echo command dsget --uri $uri --filename $workdir/$exp_id.$class_id.fits --bytes $bytes --md5 $md5sum + command dsget --uri $uri --filename $workdir/$exp_id.$class_id.fits --bytes $bytes --md5 $md5sum + end + + # success + task.exit 0 + # could run copydone from here but how would the exit status be + # handled? + + list word -split $options:0 + $exp_id = $word:0 + $camera = $word:1 + $telescope = $word:2 + $file_id = $word:3 + $bytes = $word:4 + $md5sum = $word:5 + $class = $word:6 + $class_id = $word:7 + $uri = $word:8 + $exp_tag = $word:9 + queuepush pzCopyDone -key 1:2:3:7:8 "NEW $options:0 $workdir/$exp_id.$class_id.fits" + queuepush pzPendingExp -replace -key 1:2:3:7:8 "DONE $options:0" + end + + task.exit default + queuepush pzPendingExp -replace -key 1:2:3:7:8 "FAIL $options:0" + end + + task.exit timeout + queuepush pzPendingExp -replace -key 1:2:3:7:8 "TIMEOUT $options:0" + end +end + +task pztool.copydone + periods -exec 5 + periods -poll 5 + periods -timeout 15 + + host local + + task.exec + queuesize pzCopyDone -var N + if ($N == 0) break + + # get a line out of the queue + queuepop pzCopyDone -var line -key 0 NEW + + # take the state out of the line + strpop line state + + # put the line back into the queue with a stea of RUN + queuepush pzCopyDone -replace -key 1:2:3:7:8 "RUN $line" + + # the STATE has already been removed from the line + list word -split $line + $exp_id = $word:0 + $camera = $word:1 + $telescope = $word:2 + $file_id = $word:3 + $bytes = $word:4 + $md5sum = $word:5 + $class = $word:6 + $class_id = $word:7 + $uri = $word:8 + $exp_tag = $word:9 + $new_uri = $word:10 + + # store the current line without the state + options "$line" + + echo command pztool -copydone -exp_id $exp_id -inst $camera -telescope $telescope -class $class -class_id $class_id -uri $uri + command pztool -copydone -exp_id $exp_id -inst $camera -telescope $telescope -class $class -class_id $class_id -uri $new_uri + end + + # success + task.exit 0 + queuepush pzCopyDone -replace -key 1:2:3:6:7 "DONE $options:0" + end + + task.exit default + queuepush pzCopyDone -replace -key 1:2:3:6:7 "FAIL $options:0" + end + + task.exit timeout + queuepush pzCopyDone -replace -key 1:2:3:6:7 "TIMEOUT $options:0" + end +end Index: /branches/eam_branch_00/ippTasks/testscript.pro =================================================================== --- /branches/eam_branch_00/ippTasks/testscript.pro (revision 10761) +++ /branches/eam_branch_00/ippTasks/testscript.pro (revision 10761) @@ -0,0 +1,80 @@ +# test for pcontrol +macro test.pcontrol.simple + stop + for i 0 100 + job ls -R /etc + end + + status + host add alala + + status + run +end + +# test for pcontrol +macro test.pcontrol + exec p0tool -pendingimfile -simple | head -n 100 > list.dat + + stop + list line -x "cat list.dat" + echo $line:0 + for i 0 $line:n + list word -split $line:$i + job phase0_imfile.pl --exp_tag $word:0 --class_id fpa --uri $word:3 + end + + status + host add sn2 + host add sn3 + host add sn4 + host add sn5 + + status + run +end + +# test for pclient +macro test.pclient + exec p0tool -pendingimfile -simple | head -n 100 > list.dat + + list line -x "cat list.dat" + echo $line:0 + for i 0 $line:n + list word -split $line:$i + job phase0_imfile.pl --exp_tag $word:0 --class_id fpa --uri $word:3 + + for i 0 50 + check + output /dev/null + status + output stdout + if ($JOBSTATUS == EXIT) + $i = 50 + reset + echo "DONE" + else + usleep 200000 + end + end + if ($JOBSTATUS != EXIT) + echo TIMEOUT*** + end + end +end + +macro simple.test + for j 0 100 + job ls + for i 0 50 + check + status + if ($JOBSTATUS == EXIT) + $i = 50 + reset + else + usleep 10000 + end + end + end +end