Index: branches/czw_branch/20101203/ippTasks/nightly_science.pro =================================================================== --- branches/czw_branch/20101203/ippTasks/nightly_science.pro (revision 30017) +++ branches/czw_branch/20101203/ippTasks/nightly_science.pro (revision 30017) @@ -0,0 +1,686 @@ +## nightly_science.pro : -*- sh -*- + +check.globals + +macro nightly.science.on + ns.initday.on + ns.detrends.off + ns.dqstats.on +# ns.sweetspot.on +# ns.registration.on +# ns.burntool.on +# ns.chips.on + ns.stacks.on + ns.diffs.on +end + +macro nightly.science.off + ns.initday.off + ns.detrends.off + ns.dqstats.off +# ns.sweetspot.off +# ns.registration.off +# ns.burntool.off +# ns.chips.off + ns.stacks.off + ns.diffs.off +end + +macro ns.on + nightly.science.on +end + +macro ns.off + nightly.science.off +end + +macro ns.initday.on + task ns.initday.load + active true + end +end + +macro ns.detrends.on + task ns.detrends.load + active true + end +end + +macro ns.dqstats.on + task ns.dqstats.load + active true + end +end + +# macro ns.sweetspot.on +# task ns.sweetspot.load +# active true +# end +# end + +# macro ns.registration.on +# task ns.registration.load +# active true +# end +# end + +# macro ns.burntool.on +# task ns.burntool.load +# active true +# end +# task ns.burntool.run +# active true +# end +# end + +# macro ns.chips.on +# task ns.chips.load +# active true +# end +# task ns.chips.run +# active true +# end +# end + +macro ns.stacks.on + task ns.stacks.load + active false + end + task ns.stacks.run + active true + end + task ns.stacks.confirm + active true + end +end + +macro ns.diffs.on + task ns.diffs.load + active false + end + task ns.diffs.run + active true + end +end + +macro ns.initday.off + task ns.initday.load + active false + end +end + +macro ns.detrends.off + task ns.detrends.load + active false + end +end + +macro ns.dqstats.off + task ns.dqstats.load + active false + end +end + +# macro ns.sweetspot.off +# task ns.sweetspot.load +# active false +# end +# end + +# macro ns.registration.off +# task ns.registration.load +# active false +# end +# end + +# macro ns.burntool.off +# task ns.burntool.load +# active false +# end +# task ns.burntool.run +# active false +# end +# end + +# macro ns.chips.off +# task ns.chips.load +# active false +# end +# task ns.chips.run +# active false +# end +# end + +macro ns.stacks.off + task ns.stacks.load + active false + end + task ns.stacks.run + active false + end +end + +macro ns.diffs.off + task ns.diffs.load + active false + end + task ns.diffs.run + active false + end +end + +# $ns_regPAGE = 0 +# $ns_burnPAGE = 0 +# $ns_RburnPAGE = 0 +# $ns_RburnCELL = 0 +# $ns_chipPAGE = 0 +# $ns_RchipPAGE = 0 +$ns_stackPAGE = 0 +$ns_RstackPAGE = 0 +$ns_CstackPAGE = 0 +$ns_diffPAGE = 0 +$ns_RdiffPAGE = 0 + +book init nsData +# +# Macros to control the book. +# +macro ns.add.date + book newpage nsData $1 + book setword nsData $1 nsState NEW + book setword nsData $1 nsStackState TOWARP + book setword nsData $1 nsDiffState TOWARP +end + +macro ns.show.dates + book npages nsData -var Npages + for i 0 $Npages + book getpage nsData $i -var date + book getword nsData $date nsState -var state + book getword nsData $date nsStackState -var Sstate + book getword nsData $date nsDiffState -var Dstate + book getword nsData $date nsObservingState -var Ostate + echo $date $state $Sstate $Dstate $Ostate + end +end + +macro ns.del.date + book delpage nsData $1 +end + +macro ns.set.date + book setword nsData $1 nsState $2 +end + + + +# +# Start a new date to work on +# +task ns.initday.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + trange 0:00:00 1:00:00 -nmax 1 + npending 1 + + task.exec + stdout $LOGDIR/ns.initday.log + stderr $LOGDIR/ns.initday.log + $today = `date -u +%Y-%m-%d` + book newpage nsData $today + book setword nsData $today nsState NEW + book setword nsData $today nsStackState TOWARP + book setword nsData $today nsDiffState TOWARP + +# command echo $today + command nightly_science.pl --clean_old --date $today + end + + task.exit 0 + # nothing to do here? + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Queue off any possible detrend verification runs +# +task ns.detrends.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + trange 23:00:00 23:59:59 -nmax 1 + npending 1 + + task.exec + stdout $LOGDIR/ns.detrends.log + stdout $LOGDIR/ns.detrends.log + $today = `date -u +%Y-%m-%d` + + command nightly_science.pl --queue_detrends --date $today + end + + task.exit 0 + # nothign to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout + showcommand timeout + end +end + +# +# Queue dqstats runs +# +task ns.dqstats.load + host local + periods -poll 3600 + periods -exec $LOADEXEC + periods -timeout 30 + trange 22:00:00 23:59:59 + trange 00:00:00 10:00:00 + npending 1 + + task.exec + stdout $LOGDIR/ns.dqstats.log + stderr $LOGDIR/ns.dqstats.log + $today = `date -u +%Y-%m-%d` + + command nightly_science.pl --queue_dqstats --date $today + end + + task.exit 0 + # nothing to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the warps are finished +# +task ns.stacks.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.stacks.log + + book getpage nsData $ns_stackPAGE -var date + book getword nsData $date nsStackState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.stacks.load: " $ns_stackPAGE $date $ns_STATE $Npages + end + + $ns_stackPAGE ++ + if ($ns_stackPAGE >= $Npages) set ns_stackPAGE = 0 + option $date + + if ("$ns_STATE" != "TOWARP") break + $run = nightly_science.pl --check_stacks --date $date + command $run + end + + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the warps are finished and so we can make stacks. +# + +task ns.stacks.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.stacks.log + + book getpage nsData $ns_RstackPAGE -var date + book getword nsData $date nsStackState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.stacks.run: " $ns_RstackPAGE $date $ns_STATE $Npages + end + + $ns_RstackPAGE ++ + if ($ns_RstackPAGE >= $Npages) set ns_RstackPAGE = 0 + option $date + + if (("$ns_STATE" != "QUEUESTACKS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break + $run = nightly_science.pl --queue_stacks --date $date + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 +# We've queued up stacking. We're done with this date, so don't reload the page. + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Confirm that all the stacks that can be built have been built, or at least attempted. +# +task ns.stacks.confirm + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.stacks.log + + book getpage nsData $ns_CstackPAGE -var date + book getword nsData $date nsStackState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.stacks.confirm: " $ns_CstackPAGE $date $ns_STATE $Npages + end + + $ns_CstackPAGE ++ + if ($ns_CstackPAGE >= $Npages) set ns_CstackPAGE = 0 + option $date + + if ("$ns_STATE" != "STACKING") break + $run = nightly_science.pl --confirm_stacks --date $date + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + + +# +# Check to see if the warps are finished +# +task ns.diffs.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.diffs.log + + book getpage nsData $ns_diffPAGE -var date + book getword nsData $date nsDiffState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.diffs.load: " $ns_diffPAGE $date $ns_STATE $Npages + end + + $ns_diffPAGE ++ + if ($ns_diffPAGE >= $Npages) set ns_diffPAGE = 0 + option $date + + if ("$ns_STATE" != "TOWARP") break + $run = nightly_science.pl --check_diffs --date $date + command $run + end + + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the warps are finished and so we can make diffs. +# + +task ns.diffs.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.diffs.log + + book getpage nsData $ns_RdiffPAGE -var date + book getword nsData $date nsDiffState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.diffs.run: " $ns_RdiffPAGE $date $ns_STATE $Npages + end + + $ns_RdiffPAGE ++ + if ($ns_RdiffPAGE >= $Npages) set ns_RdiffPAGE = 0 + option $date + + if (("$ns_STATE" != "QUEUEDIFFS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break + $run = nightly_science.pl --queue_diffs --date $date + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 +# We've queued up diffing. We're done with this date, so don't reload the page. + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + + + +# # +# # Queue sweetspot runs +# # +# task ns.sweetspot.load +# host local +# periods -poll 3600 +# periods -exec $LOADEXEC +# periods -timeout 30 +# trange 02:00:00 02:59:59 -nmax 1 +# npending 1 + +# task.exec +# stdout $LOGDIR/ns.sweetspot.log +# stderr $LOGDIR/ns.sweetspot.log +# $today = `date -u +%Y-%m-%d` + +# command nightly_science.pl --queue_sweetspot --date $today +# end + +# task.exit 0 +# # nothing to do here +# end +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# # operation times out +# task.exit timeout +# showcommand timeout +# end +# end +