Index: branches/eam_branches/ps2-tc3-20130727/ippTasks/Makefile.am =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTasks/Makefile.am (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTasks/Makefile.am (revision 37403) @@ -49,4 +49,5 @@ vp.pro \ bg.regeneration.pro \ + remote.pro \ fullforce.pro Index: branches/eam_branches/ps2-tc3-20130727/ippTasks/diff.pro =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTasks/diff.pro (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTasks/diff.pro (revision 37403) @@ -61,4 +61,22 @@ end task diff.revert + active false + end +end + +macro diff.summary.on + task diff.summary.load + active true + end + task diff.summary.run + active true + end +end + +macro diff.summary.off + task diff.summary.load + active false + end + task diff.summary.run active false end @@ -125,4 +143,9 @@ add_poll_args run add_poll_labels run + # increase the poll limit for warp over the default to + # help it keep up with chip processing + # NOTE : it is not a problem for difftool to have multiple + # -limit entries: the last one is used + $run = $run -limit {$POLL_LIMIT * 2} command $run end Index: branches/eam_branches/ps2-tc3-20130727/ippTasks/ipphosts.mhpcc.config =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTasks/ipphosts.mhpcc.config (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTasks/ipphosts.mhpcc.config (revision 37403) @@ -18,5 +18,5 @@ ipphosts METADATA camera STR skycell - count S32 30 + count S32 60 sky00 STR stsci00.0 sky01 STR stsci00.1 @@ -49,4 +49,34 @@ sky28 STR stsci09.1 sky29 STR stsci09.2 + sky30 STR stsci10.0 + sky31 STR stsci10.1 + sky32 STR stsci10.2 + sky33 STR stsci11.0 + sky34 STR stsci11.1 + sky35 STR stsci11.2 + sky36 STR stsci12.0 + sky37 STR stsci12.1 + sky38 STR stsci12.2 + sky39 STR stsci13.0 + sky40 STR stsci13.1 + sky41 STR stsci13.2 + sky42 STR stsci14.0 + sky43 STR stsci14.1 + sky44 STR stsci14.2 + sky45 STR stsci15.0 + sky46 STR stsci15.1 + sky47 STR stsci15.2 + sky48 STR stsci16.0 + sky49 STR stsci16.1 + sky50 STR stsci16.2 + sky51 STR stsci17.0 + sky52 STR stsci17.1 + sky53 STR stsci17.2 + sky54 STR stsci18.0 + sky55 STR stsci18.1 + sky56 STR stsci18.2 + sky57 STR stsci19.0 + sky58 STR stsci19.1 + sky59 STR stsci19.2 END Index: branches/eam_branches/ps2-tc3-20130727/ippTasks/pantasks.pro =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTasks/pantasks.pro (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTasks/pantasks.pro (revision 37403) @@ -645,4 +645,5 @@ host anyhost else + # echo host $host host $host end @@ -655,5 +656,5 @@ end - local skycellID varname count host skyname skyhash + local skycellID varname count host skyname skyhash fullhost $skycellID = $1 @@ -676,9 +677,12 @@ # missing ipphosts table results in host = NULL - book getword ipphosts skycell $skyname -var host + book getword ipphosts skycell $skyname -var fullhost + list word -splitbychar . $fullhost + $host = $word:0 if ("$host" == "NULL") host anyhost else + # echo host $host host $host end Index: branches/eam_branches/ps2-tc3-20130727/ippTasks/pstamp.pro =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTasks/pstamp.pro (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTasks/pstamp.pro (revision 37403) @@ -175,4 +175,52 @@ macro pstamp.find.off task pstamp.request.find + active false + end +end +macro pstamp.parse.on + task pstamp.request.load + active true + end + task pstamp.request.run + active true + end +end +macro pstamp.parse.off + task pstamp.request.load + active false + end + task pstamp.request.run + active false + end +end +macro pstamp.job.on + task pstamp.job.load + active true + end + task pstamp.job.run + active true + end +end +macro pstamp.job.off + task pstamp.job.load + active false + end + task pstamp.job.run + active false + end +end +macro pstamp.finish.on + task pstamp.finish.load + active true + end + task pstamp.finish.run + active true + end +end +macro pstamp.finish.off + task pstamp.finish.load + active false + end + task pstamp.finish.run active false end @@ -469,4 +517,9 @@ add_poll_args run add_poll_labels run + # limit query for finished requests to small number + # because we only run 4 at a time. Using the default queue + # depth fouls up the priority orer + # XXX: use a varaible + $run = $run -limit 8 command $run end @@ -499,4 +552,5 @@ periods -timeout 2400 host anyhost + npending 4 task.exec @@ -951,5 +1005,5 @@ # overloading nfs - npending 5 + npending 2 task.exec Index: branches/eam_branches/ps2-tc3-20130727/ippTasks/remote.pro =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTasks/remote.pro (revision 37403) +++ branches/eam_branches/ps2-tc3-20130727/ippTasks/remote.pro (revision 37403) @@ -0,0 +1,489 @@ +## remote.pro : -*- sh -*- + +check.globals + +# define chips + +book init remotePrepRuns +book init remoteExecRuns +book init remotePollRuns + +$remote_label_iter = 0 +$remote_label_iter2 = 0 +$remote_stage_iter = 0 +$remoteP_DB = 0 +$remoteE_DB = 0 +$remoteL_DB = 0 + +list STAGES + chip + camera +# warp + stack +end + +macro remote.off + task remote.define + active false + end + task remote.define.warp + active false + end + task remote.prep.load + active false + end + task remote.prep.run + active false + end + task remote.exec.load + active false + end + task remote.exec.run + active false + end + task remote.poll.load + active false + end + task remote.poll.run + active false + end +end + +macro remote.on + task remote.define + active true + end + task remote.define.warp + active true + end + task remote.prep.load + active true + end + task remote.prep.run + active true + end + task remote.exec.load + active true + end + task remote.exec.run + active true + end + task remote.poll.load + active true + end + task remote.poll.run + active true + end +end + + +task remote.define + host local + periods -poll $LOADPOLL + periods -exec 300 + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/remote.define.chip + + $label = $LABEL:$remote_label_iter + + $stage = $STAGES:$remote_stage_iter + $remote_stage_iter ++ + + if ($remote_stage_iter >= $STAGES:n) + set remote_stage_iter = 0 + $remote_label_iter ++ + if ($remote_label_iter >= $LABEL:n) set remote_label_iter = 0 + echo $remote_stage_iter $remote_label_iter $label $stage + end + + $run = remotetool -definebyquery -label $label -stage $stage -path_base neb://@HOST@.0/remote/$label -limit 500 + if ($DB:n == 0) + option DEFAULT + else + option $DB:$remoteP_DB + $run = $run -dbname $DB:$remoteP_DB + $remoteP_DB ++ + if ($remoteP_DB >= $DB:n) set remoteP_DB = 0 + end + + echo $run + command $run + + end + # success + task.exit 0 + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task remote.define.warp + host local + periods -poll $LOADPOLL + periods -exec 1800 + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/remote.define.chip + + $label = $LABEL:$remote_label_iter2 + + if ($remote_label_iter >= $LABEL:n) set remote_label_iter = 0 + echo $remote_label_iter $label warp + + $run = remotetool -definebyquery -label $label -stage warp -path_base neb://@HOST@.0/remote/$label -limit 500 + if ($DB:n == 0) + option DEFAULT + else + option $DB:$remoteP_DB + $run = $run -dbname $DB:$remoteP_DB + $remoteP_DB ++ + if ($remoteP_DB >= $DB:n) set remoteP_DB = 0 + end + + echo $run + command $run + + end + # success + task.exit 0 + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task remote.prep.load + host local + periods -poll $LOADPOLL + periods -exec 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/remote.prep.load + + $run = remotetool -listrun -state new + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$remoteP_DB + $run = $run -dbname $DB:$remoteP_DB + $remoteP_DB ++ + if ($remoteP_DB >= $DB:n) set remoteP_DB = 0 + end + + add_poll_labels run + command $run + end + # success + task.exit 0 + ipptool2book stdout remotePrepRuns -uniq -key remote_id -setword dbname $options:0 -setword pantaskState INIT + process_cleanup remotePrepRuns + + if ($VERBOSE > 2) + book listbook remotePrepRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task remote.prep.run + # this probably shouldn't be local + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 600000 + active true + npending 10 + + task.exec + stdout $LOGDIR/remote.prep.run + stderr $LOGDIR/remote.prep.run + + book npages remotePrepRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + + book getpage remotePrepRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword remotePrepRuns $pageName pantaskState RUN + book getword remotePrepRuns $pageName remote_id -var REMOTE_ID + book getword remotePrepRuns $pageName stage -var STAGE + book getword remotePrepRuns $pageName path_base -var PATH_BASE +# book getword remotePrepRuns $pageName label -var LABEL + book getword remotePrepRuns $pageName dbname -var DBNAME + + sprintf outroot "%s/remote_%s.%s" $PATH_BASE $STAGE $REMOTE_ID + + if ("$STAGE" == "chip") + $command = sc_prepare_chip.pl --camera GPC1 --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + end + if ("$STAGE" == "camera") + $command = sc_prepare_camera.pl --camera GPC1 --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + end + if ("$STAGE" == "warp") + $command = sc_prepare_warp.pl --camera GPC1 --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + end + if ("$STAGE" == "stack") + $command = sc_prepare_stack.pl --camera GPC1 --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + end + + options $pageName + command $command + end + + # default exit status + task.exit default + process_exit remotePrepRuns $options:0 $JOB_STATUS + end + # locked list + task.exit crash + process_exit remotePrepRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword remotePrepRuns $options:0 pantaskState TIMEOUT + end +end + + +task remote.exec.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/remote.exec.load + +# $end_date = `date +%Y-%m-%dT%H:%M:%S` + $run = remotetool -listrun -state pending +# -poll_end $end_date + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$remoteE_DB + $run = $run -dbname $DB:$remoteE_DB + $remoteE_DB ++ + if ($remoteE_DB >= $DB:n) set remoteE_DB = 0 + end + + add_poll_labels run + command $run + end + # success + task.exit 0 + ipptool2book stdout remoteExecRuns -uniq -key remote_id -setword dbname $options:0 -setword pantaskState INIT + process_cleanup remoteExecRuns + + if ($VERBOSE > 2) + book listbook remoteExecRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task remote.exec.run + # this probably shouldn't be local + host local + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 6000000 + active true + npending 3 + + task.exec + stdout $LOGDIR/remote.exec.run + stderr $LOGDIR/remote.exec.run + + book npages remoteExecRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + + book getpage remoteExecRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword remoteExecRuns $pageName pantaskState RUN + book getword remoteExecRuns $pageName remote_id -var REMOTE_ID + book getword remoteExecRuns $pageName stage -var STAGE + book getword remoteExecRuns $pageName path_base -var PATH_BASE +# book getword remoteExecRuns $pageName label -var LABEL + book getword remoteExecRuns $pageName job_id -var JOB_ID + book getword remoteExecRuns $pageName dbname -var DBNAME + + sprintf outroot "%s/remote_%s.%s" $PATH_BASE $STAGE $REMOTE_ID + + $command = sc_remote_exec.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 + + options $pageName + command $command + end + + # default exit status + task.exit default + process_exit remoteExecRuns $options:0 $JOB_STATUS + end + # locked list + task.exit crash + process_exit remoteExecRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword remoteExecRuns $options:0 pantaskState TIMEOUT + end +end + + +task remote.poll.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/remote.poll.load + + $end_date = `date +%Y-%m-%dT%H:%M:%S -d '1 hour ago'` + $run = remotetool -listrun -state run -poll_end $end_date + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$remoteL_DB + $run = $run -dbname $DB:$remoteL_DB + $remoteL_DB ++ + if ($remoteL_DB >= $DB:n) set remoteL_DB = 0 + end + + add_poll_labels run + command $run + end + # success + task.exit 0 + ipptool2book stdout remotePollRuns -uniq -key remote_id -setword dbname $options:0 -setword pantaskState INIT + process_cleanup remotePollRuns + + if ($VERBOSE > 2) + book listbook remotePollRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task remote.poll.run + # this probably shouldn't be local + host local + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 6000000 + active true + npending 3 + + task.exec + stdout $LOGDIR/remote.poll.run + stderr $LOGDIR/remote.poll.run + + book npages remotePollRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + + book getpage remotePollRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword remotePollRuns $pageName pantaskState RUN + book getword remotePollRuns $pageName remote_id -var REMOTE_ID + book getword remotePollRuns $pageName stage -var STAGE + book getword remotePollRuns $pageName path_base -var PATH_BASE +# book getword remotePollRuns $pageName label -var LABEL + book getword remotePollRuns $pageName job_id -var JOB_ID + book getword remotePollRuns $pageName dbname -var DBNAME + + sprintf outroot "%s/remote_%s.%s" $PATH_BASE $STAGE $REMOTE_ID + + # This can't have an invalid job_id + if ($JOB_ID == -1) break + $command = sc_remote_exec.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 --poll --job_id $JOB_ID + + options $pageName + command $command + end + + # default exit status + task.exit default + process_exit remotePollRuns $options:0 $JOB_STATUS + end + # locked list + task.exit crash + process_exit remotePollRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword remotePollRuns $options:0 pantaskState TIMEOUT + end +end + Index: branches/eam_branches/ps2-tc3-20130727/ippTasks/stack.pro =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTasks/stack.pro (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTasks/stack.pro (revision 37403) @@ -72,4 +72,22 @@ active false end + task stack.summary.load + active false + end + task stack.summary.run + active false + end +end + +macro stack.summary.on + task stack.summary.load + active true + end + task stack.summary.run + active true + end +end + +macro stack.summary.off task stack.summary.load active false Index: branches/eam_branches/ps2-tc3-20130727/ippTasks/summit.copy.pro =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTasks/summit.copy.pro (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTasks/summit.copy.pro (revision 37403) @@ -567,8 +567,13 @@ command pztool -clearcommonfaults -dbname $DB:$pztoolClearFault_DB + # loop over the multiple databases quickly, then pause for 5 min $pztoolClearFault_DB ++ - if ($pztoolClearFault_DB >= $DB:n) set pztoolClearFault_DB = 0 + if ($pztoolClearFault_DB >= $DB:n) + set pztoolClearFault_DB = 0 + periods -exec 300 + else + periods -exec 30 + end end - periods -exec 1800 end Index: branches/eam_branches/ps2-tc3-20130727/ippTasks/survey.pro =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTasks/survey.pro (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTasks/survey.pro (revision 37403) @@ -668,5 +668,4 @@ end -# echo $run command $run end @@ -698,4 +697,7 @@ periods -exec $SURVEY_EXEC periods -timeout $SURVEY_TIMEOUT + ## time in UT -- want to start ~0900 HST and run until next night starts. 24h boundary so two trange needed? + trange 19:00:00 23:59:00 + trange 00:00:00 05:00:00 npending 1 @@ -752,5 +754,4 @@ end -# echo $run command $run end @@ -838,5 +839,4 @@ end - # echo $run command $run end @@ -1437,6 +1437,8 @@ host local - periods -poll $SURVEY_POLL - periods -exec $SURVEY_EXEC + # we are going to use a long timeout (900 sec) between passes of the list + # and a short timeout (10 sec) while we are doing the full list + periods -exec 10 + periods -poll 2 periods -timeout $SURVEY_TIMEOUT npending 1 @@ -1449,7 +1451,10 @@ book npages SURVEY_PUBLISH -var N if ($N == 0) - # echo "No labels for processing" + # echo "No labels for processing" break endif + + # short timeout while we have something to do + periods -exec 10 # survey.publish allows multiple entries per label. @@ -1465,7 +1470,10 @@ book getpage SURVEY_PUBLISH 0 -var tag -key STATE NEW - # Select different database - $SURVEY_PUBLISH ++ - if ($SURVEY_PUBLISH >= $DB:n) set SURVEY_PUBLISH = 0 + # Select different database, set a long timeout + $SURVEY_PUBLISH_DB ++ + if ($SURVEY_PUBLISH_DB >= $DB:n) set SURVEY_PUBLISH_DB = 0 + periods -exec 900 + date -var mytime + echo "done with publish list @ $mytime" end @@ -1484,10 +1492,9 @@ option DEFAULT else - $run = $run -dbname $DB:$SURVEY_PUBLISH - option $DB:$SURVEY_PUBLISH + $run = $run -dbname $DB:$SURVEY_PUBLISH_DB + option $DB:$SURVEY_PUBLISH_DB end # echo $run - command $run @@ -1859,5 +1866,4 @@ end - echo $run command $run end Index: branches/eam_branches/ps2-tc3-20130727/ippTasks/warp.pro =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTasks/warp.pro (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTasks/warp.pro (revision 37403) @@ -99,4 +99,22 @@ end task warp.revert.overlap + active false + end +end + +macro warp.summary.on + task warp.summary.load + active true + end + task warp.summary.run + active true + end +end + +macro warp.summary.off + task warp.summary.load + active false + end + task warp.summary.run active false end @@ -261,4 +279,9 @@ add_poll_args run add_poll_labels run + # increase the poll limit for warp over the default to + # help it keep up with chip processing + # NOTE : it is not a problem for warptool to have multiple + # -limit entries: the last one is used + $run = $run -limit {$POLL_LIMIT * 2} command $run end