Index: /tags/ipp-20141024/ippTasks/nightly_science.pro =================================================================== --- /tags/ipp-20141024/ippTasks/nightly_science.pro (revision 38242) +++ /tags/ipp-20141024/ippTasks/nightly_science.pro (revision 38243) @@ -5,5 +5,5 @@ macro nightly.science.on ns.initday.on - ns.detrends.off +# ns.detrends.off ns.dqstats.on ns.stacks.on @@ -13,5 +13,5 @@ macro nightly.science.off ns.initday.off - ns.detrends.off +# ns.detrends.off ns.dqstats.off ns.stacks.off @@ -33,9 +33,4 @@ end -macro ns.detrends.on - task ns.detrends.load - active true - end -end macro ns.dqstats.on @@ -46,19 +41,10 @@ macro ns.stacks.on - task ns.stacks.load - active false - end task ns.stacks.run active true end - task ns.stacks.confirm - active false - end end macro ns.diffs.on - task ns.diffs.load - active false - end task ns.diffs.run active true @@ -72,10 +58,4 @@ end -macro ns.detrends.off - task ns.detrends.load - active false - end -end - macro ns.dqstats.off task ns.dqstats.load @@ -85,33 +65,11 @@ macro ns.stacks.off - task ns.stacks.load - active false - end task ns.stacks.run active false end - task ns.stacks.confirm - active false - end - end macro ns.diffs.off - task ns.diffs.load - active false - end task ns.diffs.run - active false - end -end - -macro ns.stacks.confirm.on - task ns.stacks.confirm - active true - end -end - -macro ns.stacks.confirm.off - task ns.stacks.confirm active false end @@ -128,23 +86,52 @@ book init nsDiffs book init nsStacks + +# Hard coded values +$camera_map:gpc1 = "GPC1" +$camera_map:gpc2 = "GPC2" + +# Macro to manage the camera maps +macro ns.add.camera_map + if ($0 != 3) + echo "USAGE: ns.add.camera_map (dbname) (camera)" + break + end + + $camera_map:$1 = $2 +end + # # Macros to control the book. # macro ns.add.date - if ($0 != 2) - echo "USAGE: ns.add.date (YYYY-MM-DD)" + if ($0 != 4) + echo "USAGE: ns.add.date (YYYY-MM-DD) (dbname) (camera)" break end - book newpage nsData $1 - book setword nsData $1 nsState NEW - book setword nsData $1 dbname $DB:0 - - book newpage nsDiffs $1 - book setword nsDiffs $1 nsDiffState TOWARP - book setword nsDiffs $1 nsObservingState UNKNOWN - - book newpage nsStacks $1 - book setword nsStacks $1 nsStackState TOWARP + if ($?camera_map:$2) + if ($camera_map:$2 != "$3") + echo "camera $3 does not match expected camera $camera_map:$2" + break + end + else + echo "dbname $2 does not have a camera_map entry" + break + end + + # build key from date and database name + sprintf key "%s-%s" $1 $2 + book newpage nsData $key + book setword nsData $key nsState NEW + book setword nsData $key date $1 + book setword nsData $key dbname $2 + book setword nsData $key camera $3 + + book newpage nsDiffs $key + book setword nsDiffs $key nsDiffState TOWARP + book setword nsDiffs $key nsObservingState UNKNOWN + + book newpage nsStacks $key + book setword nsStacks $key nsStackState TOWARP end @@ -153,10 +140,15 @@ book npages nsData -var Npages for i 0 $Npages - book getpage nsData $i -var date - book getword nsData $date nsState -var state - book getword nsStacks $date nsStackState -var Sstate - book getword nsDiffs $date nsDiffState -var Dstate - book getword nsDiffs $date nsObservingState -var Ostate - echo $date $state $Sstate $Dstate $Ostate + book getpage nsData $i -var key + book getword nsData $key date -var date + book getword nsData $key dbname -var dbname + book getword nsData $key camera -var camera + book getword nsData $key nsState -var state + book getword nsStacks $key nsStackState -var Sstate + book getword nsDiffs $key nsDiffState -var Dstate + book getword nsDiffs $key nsObservingState -var Ostate +# I've moved dbname to end as czartool reads information from here, and it may be fixed length + echo $key $date $state $Sstate $Dstate $Ostate $dbname $camera + end end @@ -179,5 +171,5 @@ periods -exec $LOADEXEC periods -timeout 300 - trange 0:00:00 1:00:00 -nmax 1 + trange 0:00:00 1:00:00 -nmax 2 npending 1 @@ -186,16 +178,25 @@ stderr $LOGDIR/ns.initday.log $today = `date -u +%Y-%m-%d` - book newpage nsData $today - book setword nsData $today nsState NEW - book newpage nsStacks $today - book setword nsStacks $today nsStackState TOWARP - book newpage nsDiffs $today - book setword nsDiffs $today nsDiffState TOWARP - book setword nsData $today dbname $DB:0 - -# command echo $today - $run = nightly_science.pl --clean_old --date $today --dbname $DB:0 -# add_standard_args run - command $run + # loop over all databases + for i 0 $DB:n + $thisDB = $DB:$i + sprintf key "%s-%s" $today $thisDB + book newpage nsData $key + book setword nsData $key date $today + book setword nsData $key nsState NEW + book setword nsData $key dbname $thisDB + book setword nsData $key camera $camera_map:$thisDB + + book newpage nsStacks $key + book setword nsStacks $key date $today + book setword nsStacks $key nsStackState TOWARP + book newpage nsDiffs $key + book setword nsDiffs $key date $today + book setword nsDiffs $key nsDiffState TOWARP + + #command echo $today + $run = nightly_science.pl --clean_old --date $today --dbname $thisDB --camera $camera_map:$thisDB + command $run + end end @@ -217,43 +218,14 @@ # -# Queue off any possible detrend verification runs -# -task ns.detrends.load - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - trange 23:00:00 23:59:59 -nmax 1 - npending 1 - - task.exec - stdout $LOGDIR/ns.detrends.log - stdout $LOGDIR/ns.detrends.log - $today = `date -u +%Y-%m-%d` - - $run = nightly_science.pl --queue_detrends --date $today --dbname $DB:0 -# add_standard_args $run - command $run - end - - task.exit 0 - # nothign to do here - end - # locked list - task.exit default - showcommand failure - end - task.exit crash - showcommand crash - end - # operation times out - task.exit timeout - showcommand timeout - end -end - -# # Queue dqstats runs # + +# CZW: 2015-05-05 One day we will stop making these, but that day is not today. +# According to Ken, we do not need to implement this for gpc2, as nothing is +# written on the OTIS/scheduler side to accept these products. We should +# continue to generate these for gpc1, as the machinery does exist to check +# for them, and not generating them may cause that to fail (even if nothing +# is done with the results). + task ns.dqstats.load host local @@ -270,5 +242,5 @@ $today = `date -u +%Y-%m-%d` - $run = nightly_science.pl --queue_dqstats --date $today --dbname $DB:0 + $run = nightly_science.pl --queue_dqstats --date $today --dbname gpc1 --camera GPC1 # add_standard_args $run command $run @@ -288,74 +260,4 @@ # operation times out task.exit timeout - showcommand timeout - end -end - -# -# Check to see if the warps are finished -# -task ns.stacks.load - active false - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 120 - npending 1 - - task.exec - stdout NULL - stderr $LOGDIR/ns.stacks.log - - book getpage nsData $ns_stackPAGE -var date - book getword nsStacks $date nsStackState -var ns_STATE - book getword nsDiffs $date nsDiffState -var ns_diff_STATE - book getword nsData $date dbname -var DBNAME - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.stacks.load: " $ns_stackPAGE $date $ns_STATE $Npages - end - - $ns_stackPAGE ++ - if ($ns_stackPAGE >= $Npages) set ns_stackPAGE = 0 - option $date - - if ("$ns_STATE" != "TOWARP") break - - $run = nightly_science.pl --check_stacks --date $date --dbname $DBNAME -# add_standard_args $run - command $run - - end - - # success - task.exit 0 -# convert 'stdout' to book format - book delpage nsStacks $options:0 - ipptool2book stdout nsStacks -uniq -key date - - book getword nsStacks $options:0 nsNmacros -var ns_Nmacros - if ("$ns_Nmacros" != "NULL") - for i 0 $ns_Nmacros - sprintf macroName "ns%dMacro" $i - book getword nsStacks $options:0 $macroName -var macroCmd - $macroCmd - end - end - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - showcommand failure - end - task.exit crash - showcommand crash - end - #operation times out? - task.exit timeout showcommand timeout end @@ -377,20 +279,22 @@ stderr $LOGDIR/ns.stacks.log - book getpage nsData $ns_RstackPAGE -var date - book getword nsStacks $date nsStackState -var ns_STATE - book getword nsDiffs $date nsDiffState -var ns_diff_STATE - book getword nsData $date dbname -var DBNAME + book getpage nsData $ns_RstackPAGE -var key + book getword nsStacks $key date -var date + book getword nsStacks $key nsStackState -var ns_STATE + book getword nsDiffs $key nsDiffState -var ns_diff_STATE + book getword nsData $key dbname -var DBNAME + book getword nsData $key camera -var CAMERA book npages nsData -var Npages if ($VERBOSE > 5) - echo "ns.stacks.run: " $ns_RstackPAGE $date $ns_STATE $Npages + echo "ns.stacks.run: " $ns_RstackPAGE $key $date $ns_STATE $Npages end $ns_RstackPAGE ++ if ($ns_RstackPAGE >= $Npages) set ns_RstackPAGE = 0 - option $date + option $key if (("$ns_STATE" != "STACKING")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break - $run = nightly_science.pl --queue_stacks --date $date --dbname $DBNAME + $run = nightly_science.pl --queue_stacks --date $date --dbname $DBNAME --camera $CAMERA # add_standard_args $run command $run @@ -430,138 +334,4 @@ end -# -# Confirm that all the stacks that can be built have been built, or at least attempted. -# -task ns.stacks.confirm - active false - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 120 - npending 1 - - task.exec - stdout NULL - stderr $LOGDIR/ns.stacks.log - - book getpage nsData $ns_CstackPAGE -var date - book getword nsStacks $date nsStackState -var ns_STATE - book getword nsData $date dbname -var DBNAME - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.stacks.confirm: " $ns_CstackPAGE $date $ns_STATE $Npages - end - - $ns_CstackPAGE ++ - if ($ns_CstackPAGE >= $Npages) set ns_CstackPAGE = 0 - option $date - - if ("$ns_STATE" != "STACKING") break - $run = nightly_science.pl --confirm_stacks --date $date --dbname $DBNAME -# add_standard_args $run - command $run - end - # success - task.exit 0 -# convert 'stdout' to book format - book delpage nsStacks $options:0 - ipptool2book stdout nsStacks -uniq -key date - - book getword nsStacks $options:0 nsNmacros -var ns_Nmacros - if ("$ns_Nmacros" != "NULL") - for i 0 $ns_Nmacros - sprintf macroName "ns%dMacro" $i - book getword nsStacks $options:0 $macroName -var macroCmd - $macroCmd - end - end - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - showcommand failure - end - task.exit crash - showcommand crash - end - #operation times out? - task.exit timeout - showcommand timeout - end -end - - -# -# Check to see if the warps are finished -# -task ns.diffs.load - active false - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 120 - npending 1 - - task.exec - stdout NULL - stderr $LOGDIR/ns.diffs.log - - book getpage nsData $ns_diffPAGE -var date - book getword nsDiffs $date nsDiffState -var ns_STATE - book getword nsStacks $date nsStackState -var ns_stack_STATE - book getword nsData $date dbname -var DBNAME - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.diffs.load: " $ns_diffPAGE $date $ns_STATE $Npages - end - - $ns_diffPAGE ++ - if ($ns_diffPAGE >= $Npages) set ns_diffPAGE = 0 - option $date - - if ("$ns_STATE" != "TOWARP") break - $run = nightly_science.pl --check_diffs --date $date --dbname $DBNAME -# add_standard_args $run - command $run - end - - # success - task.exit 0 -# convert 'stdout' to book format - book delpage nsDiffs $options:0 - ipptool2book stdout nsDiffs -uniq -key date - - book getword nsDiffs $options:0 nsNmacros -var ns_Nmacros - if ("$ns_Nmacros" != "NULL") - for i 0 $ns_Nmacros - sprintf macroName "ns%dMacro" $i - book getword nsDiffs $options:0 $macroName -var macroCmd - $macroCmd - end - end - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - showcommand failure - end - task.exit crash - showcommand crash - end - #operation times out? - task.exit timeout - showcommand timeout - end -end # @@ -580,8 +350,10 @@ stderr $LOGDIR/ns.diffs.log - book getpage nsData $ns_RdiffPAGE -var date - book getword nsDiffs $date nsDiffState -var ns_STATE - book getword nsStacks $date nsStackState -var ns_stack_STATE - book getword nsData $date dbname -var DBNAME + book getpage nsData $ns_RdiffPAGE -var key + book getword nsDiffs $key date -var date + book getword nsDiffs $key nsDiffState -var ns_STATE + book getword nsStacks $key nsStackState -var ns_stack_STATE + book getword nsData $key dbname -var DBNAME + book getword nsData $key camera -var CAMERA book npages nsData -var Npages @@ -592,8 +364,8 @@ $ns_RdiffPAGE ++ if ($ns_RdiffPAGE >= $Npages) set ns_RdiffPAGE = 0 - option $date + option $key if (("$ns_STATE" != "DIFFING")&&("$ns_STATE" != "QUEUEDIFFS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break - $run = nightly_science.pl --queue_diffs --date $date --dbname $DBNAME + $run = nightly_science.pl --queue_diffs --date $date --dbname $DBNAME --camera $CAMERA # add_standard_args $run command $run @@ -635,4 +407,291 @@ +# This is the graveyard of tasks that we've defined in the past, but either never use, or never let go active. + +# if (0) + + +# macro ns.detrends.on +# task ns.detrends.load +# active true +# end +# end + + +# macro ns.detrends.off +# task ns.detrends.load +# active false +# end +# end + +# macro ns.stacks.confirm.on +# task ns.stacks.confirm +# active true +# end +# end + +# macro ns.stacks.confirm.off +# task ns.stacks.confirm +# active false +# end +# end + + + + + +# # +# # Queue off any possible detrend verification runs +# # +# task ns.detrends.load +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 30 +# trange 23:00:00 23:59:59 -nmax 1 +# npending 1 + +# task.exec +# stdout $LOGDIR/ns.detrends.log +# stdout $LOGDIR/ns.detrends.log +# $today = `date -u +%Y-%m-%d` + +# $run = nightly_science.pl --queue_detrends --date $today --dbname $DB:0 +# # add_standard_args $run +# command $run +# end + +# task.exit 0 +# # nothign to do here +# end +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# # operation times out +# task.exit timeout +# showcommand timeout +# end +# end + + +# # +# # Check to see if the warps are finished +# # +# task ns.stacks.load +# active false +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 120 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.stacks.log + +# book getpage nsData $ns_stackPAGE -var key +# book getword nsStacks $key date -var date +# book getword nsStacks $key nsStackState -var ns_STATE +# book getword nsDiffs $key nsDiffState -var ns_diff_STATE +# book getword nsData $key dbname -var DBNAME +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.stacks.load: " $ns_stackPAGE $date $DBNAME $ns_STATE $Npages +# end + +# $ns_stackPAGE ++ +# if ($ns_stackPAGE >= $Npages) set ns_stackPAGE = 0 +# option $key + +# if ("$ns_STATE" != "TOWARP") break + +# $run = nightly_science.pl --check_stacks --date $date --dbname $DBNAME +# # add_standard_args $run +# command $run + +# end + +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsStacks $options:0 +# ipptool2book stdout nsStacks -uniq -key date + +# book getword nsStacks $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsStacks $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + +# # +# # Confirm that all the stacks that can be built have been built, or at least attempted. +# # +# task ns.stacks.confirm +# active false +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 120 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.stacks.log + +# book getpage nsData $ns_CstackPAGE -var key +# book getword nsStacks $key date -var date +# book getword nsStacks $key nsStackState -var ns_STATE +# book getword nsData $key dbname -var DBNAME +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.stacks.confirm: " $ns_CstackPAGE $key $date $DBNAME $ns_STATE $Npages +# end + +# $ns_CstackPAGE ++ +# if ($ns_CstackPAGE >= $Npages) set ns_CstackPAGE = 0 +# option $key + +# if ("$ns_STATE" != "STACKING") break +# $run = nightly_science.pl --confirm_stacks --date $date --dbname $DBNAME +# # add_standard_args $run +# command $run +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsStacks $options:0 +# ipptool2book stdout nsStacks -uniq -key date + +# book getword nsStacks $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsStacks $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + + + + +# # +# # Check to see if the warps are finished +# # +# task ns.diffs.load +# active false +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 120 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.diffs.log + +# book getpage nsData $ns_diffPAGE -var key +# book getword nsDiffs $key date -var date +# book getword nsDiffs $key nsDiffState -var ns_STATE +# book getword nsStacks $key nsStackState -var ns_stack_STATE +# book getword nsData $key dbname -var DBNAME +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.diffs.load: " $ns_diffPAGE $key $date $DBNAME $ns_STATE $Npages +# end + +# $ns_diffPAGE ++ +# if ($ns_diffPAGE >= $Npages) set ns_diffPAGE = 0 +# option $key + +# if ("$ns_STATE" != "TOWARP") break +# $run = nightly_science.pl --check_diffs --date $date --dbname $DBNAME +# # add_standard_args $run +# command $run +# end + +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsDiffs $options:0 +# ipptool2book stdout nsDiffs -uniq -key date + +# book getword nsDiffs $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsDiffs $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + + +# # END GRAVEYARD +# end + + # # # # Queue sweetspot runs