Index: trunk/ippTasks/remote.pro =================================================================== --- trunk/ippTasks/remote.pro (revision 36972) +++ trunk/ippTasks/remote.pro (revision 37835) @@ -5,4 +5,5 @@ # define chips +book init remotePrepCompRuns book init remotePrepRuns book init remoteExecRuns @@ -10,9 +11,13 @@ $remote_label_iter = 0 -$remote_label_iter2 = 0 +$remote_label_warp = 0 $remote_stage_iter = 0 -$remoteP_DB = 0 -$remoteE_DB = 0 -$remoteL_DB = 0 + +$remoteDefine_DB = 0 +$remoteDefineWarp_DB = 0 +$remotePrepComp_DB = 0 +$remotePrep_DB = 0 +$remoteExec_DB = 0 +$remotePoll_DB = 0 list STAGES @@ -21,4 +26,5 @@ # warp stack + staticsky end @@ -30,4 +36,10 @@ active false end + task remote.prepcomp.load + active false + end + task remote.prepcomp.run + active false + end task remote.prep.load active false @@ -57,4 +69,10 @@ active true end + task remote.prepcomp.load + active true + end + task remote.prepcomp.run + active true + end task remote.prep.load active true @@ -76,5 +94,4 @@ end end - task remote.define @@ -83,4 +100,5 @@ periods -exec 300 periods -timeout 30 + active true npending 1 @@ -94,4 +112,5 @@ $remote_stage_iter ++ + # loop over all stages x all labels if ($remote_stage_iter >= $STAGES:n) set remote_stage_iter = 0 @@ -101,12 +120,13 @@ end + # min entry limit? $run = remotetool -definebyquery -label $label -stage $stage -path_base neb://@HOST@.0/remote/$label -limit 500 if ($DB:n == 0) option DEFAULT else - option $DB:$remoteP_DB - $run = $run -dbname $DB:$remoteP_DB - $remoteP_DB ++ - if ($remoteP_DB >= $DB:n) set remoteP_DB = 0 + option $DB:$remoteDefine_DB + $run = $run -dbname $DB:$remoteDefine_DB + $remoteDefine_DB ++ + if ($remoteDefine_DB >= $DB:n) set remoteDefine_DB = 0 end @@ -134,6 +154,7 @@ host local periods -poll $LOADPOLL - periods -exec 1800 + periods -exec 900 periods -timeout 30 + active true npending 1 @@ -142,17 +163,17 @@ stderr $LOGDIR/remote.define.chip - $label = $LABEL:$remote_label_iter2 - - if ($remote_label_iter >= $LABEL:n) set remote_label_iter = 0 - echo $remote_label_iter $label warp - - $run = remotetool -definebyquery -label $label -stage warp -path_base neb://@HOST@.0/remote/$label -limit 500 + $label = $LABEL:$remote_label_warp + + if ($remote_label_warp >= $LABEL:n) set remote_label_warp = 0 + echo $remote_label_warp $label warp + + $run = remotetool -definebyquery -label $label -stage warp -path_base neb://@HOST@.0/remote/$label -limit 50 if ($DB:n == 0) option DEFAULT else - option $DB:$remoteP_DB - $run = $run -dbname $DB:$remoteP_DB - $remoteP_DB ++ - if ($remoteP_DB >= $DB:n) set remoteP_DB = 0 + option $DB:$remoteDefineWarp_DB + $run = $run -dbname $DB:$remoteDefineWarp_DB + $remoteDefineWarp_DB ++ + if ($remoteDefineWarp_DB >= $DB:n) set remoteDefineWarp_DB = 0 end @@ -161,5 +182,6 @@ end - # success + + # success task.exit 0 end @@ -174,4 +196,123 @@ task.exit timeout showcommand timeout + end +end + +task remote.prepcomp.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + active true + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/remote.prepcomp.load + + $run = remotetool -listcomponent -state new + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$remotePrepComp_DB + $run = $run -dbname $DB:$remotePrepComp_DB + $remotePrepComp_DB ++ + if ($remotePrepComp_DB >= $DB:n) set remotePrepComp_DB = 0 + end + + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + ipptool2book stdout remotePrepCompRuns -uniq -key remote_id:stage_id -setword dbname $options:0 -setword pantaskState INIT + process_cleanup remotePrepCompRuns + + if ($VERBOSE > 2) + book listbook remotePrepCompRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the prep component operations in parallel +task remote.prepcomp.run + host anyhost + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 600 + active true + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + stdout $LOGDIR/remote.prepcomp.run + stderr $LOGDIR/remote.prepcomp.run + + book npages remotePrepCompRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + + book getpage remotePrepCompRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword remotePrepCompRuns $pageName pantaskState RUN + book getword remotePrepCompRuns $pageName remote_id -var REMOTE_ID + book getword remotePrepCompRuns $pageName stage_id -var STAGE_ID + book getword remotePrepCompRuns $pageName stage -var STAGE + book getword remotePrepCompRuns $pageName run_path_base -var RUN_PATH_BASE + book getword remotePrepCompRuns $pageName dbname -var DBNAME + + sprintf outroot "%s/remote_%s.%s/stage_%s" $RUN_PATH_BASE $STAGE $REMOTE_ID $STAGE_ID + + if ("$STAGE" == "chip") + $command = sc_prepare_chip.pl --camera GPC1 --chip_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + end + if ("$STAGE" == "camera") + $command = sc_prepare_camera.pl --camera GPC1 --cam_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + end + if ("$STAGE" == "warp") + $command = sc_prepare_warp.pl --camera GPC1 --warp_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + end + if ("$STAGE" == "stack") + $command = sc_prepare_stack.pl --camera GPC1 --stack_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + end + if ("$STAGE" == "staticsky") + $command = sc_prepare_staticsky.pl --camera GPC1 --sky_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot -dbname $DBNAME + end + + options $pageName + + periods -exec 0.05 + command $command + end + + # default exit status + task.exit default + process_exit remotePrepCompRuns $options:0 $JOB_STATUS + end + # locked list + task.exit crash + process_exit remotePrepCompRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword remotePrepCompRuns $options:0 pantaskState TIMEOUT end end @@ -181,4 +322,5 @@ periods -poll $LOADPOLL periods -exec 30 + active true npending 1 @@ -187,13 +329,13 @@ stderr $LOGDIR/remote.prep.load - $run = remotetool -listrun -state new + $run = remotetool -doneprep -state new if ($DB:n == 0) option DEFAULT else - option $DB:$remoteP_DB - $run = $run -dbname $DB:$remoteP_DB - $remoteP_DB ++ - if ($remoteP_DB >= $DB:n) set remoteP_DB = 0 + option $DB:$remotePrep_DB + $run = $run -dbname $DB:$remotePrep_DB + $remotePrep_DB ++ + if ($remotePrep_DB >= $DB:n) set remotePrep_DB = 0 end @@ -201,4 +343,5 @@ command $run end + # success task.exit 0 @@ -230,5 +373,5 @@ periods -timeout 600000 active true - npending 10 + npending 1 task.exec @@ -247,21 +390,10 @@ book getword remotePrepRuns $pageName stage -var STAGE book getword remotePrepRuns $pageName path_base -var PATH_BASE -# book getword remotePrepRuns $pageName label -var LABEL +# book getword remotePrepRuns $pageName label -var LABEL book getword remotePrepRuns $pageName dbname -var DBNAME sprintf outroot "%s/remote_%s.%s" $PATH_BASE $STAGE $REMOTE_ID - if ("$STAGE" == "chip") - $command = sc_prepare_chip.pl --camera GPC1 --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME - end - if ("$STAGE" == "camera") - $command = sc_prepare_camera.pl --camera GPC1 --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME - end - if ("$STAGE" == "warp") - $command = sc_prepare_warp.pl --camera GPC1 --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME - end - if ("$STAGE" == "stack") - $command = sc_prepare_stack.pl --camera GPC1 --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME - end + $command = sc_prepare_run.pl --camera GPC1 --remote_id $REMOTE_ID --stage $STAGE --path_base $outroot --dbname $DBNAME options $pageName @@ -284,5 +416,4 @@ end end - task remote.exec.load @@ -290,4 +421,5 @@ periods -poll $LOADPOLL periods -exec $LOADEXEC + active false npending 1 @@ -303,8 +435,8 @@ option DEFAULT else - option $DB:$remoteE_DB - $run = $run -dbname $DB:$remoteE_DB - $remoteE_DB ++ - if ($remoteE_DB >= $DB:n) set remoteE_DB = 0 + option $DB:$remoteExec_DB + $run = $run -dbname $DB:$remoteExec_DB + $remoteExec_DB ++ + if ($remoteExec_DB >= $DB:n) set remoteExec_DB = 0 end @@ -340,5 +472,5 @@ periods -exec 30 periods -timeout 6000000 - active true + active false npending 3 @@ -386,9 +518,9 @@ end - task remote.poll.load host local periods -poll $LOADPOLL - periods -exec $LOADEXEC + periods -exec 300 + active false npending 1 @@ -397,14 +529,15 @@ stderr $LOGDIR/remote.poll.load - $end_date = `date +%Y-%m-%dT%H:%M:%S -d '1 hour ago'` - $run = remotetool -listrun -state run -poll_end $end_date + # $end_date = `date +%Y-%m-%dT%H:%M:%S -d '1 hour ago'` + # $run = remotetool -listrun -state run -poll_end $end_date + $run = remotetool -listrun -state run if ($DB:n == 0) option DEFAULT else - option $DB:$remoteL_DB - $run = $run -dbname $DB:$remoteL_DB - $remoteL_DB ++ - if ($remoteL_DB >= $DB:n) set remoteL_DB = 0 + option $DB:$remotePoll_DB + $run = $run -dbname $DB:$remotePoll_DB + $remotePoll_DB ++ + if ($remotePoll_DB >= $DB:n) set remotePoll_DB = 0 end @@ -412,4 +545,5 @@ command $run end + # success task.exit 0 @@ -440,6 +574,6 @@ periods -exec 30 periods -timeout 6000000 - active true - npending 3 + active false + npending 10 task.exec @@ -466,5 +600,5 @@ # This can't have an invalid job_id if ($JOB_ID == -1) break - $command = sc_remote_exec.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 --poll --job_id $JOB_ID + $command = sc_remote_poll.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 --job_id $JOB_ID options $pageName