Changeset 38243
- Timestamp:
- May 7, 2015, 2:10:21 PM (11 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
tags/ipp-20141024/ippTasks/nightly_science.pro
- Property svn:mergeinfo set to
r35881 r38243 5 5 macro nightly.science.on 6 6 ns.initday.on 7 ns.detrends.off7 # ns.detrends.off 8 8 ns.dqstats.on 9 9 ns.stacks.on … … 13 13 macro nightly.science.off 14 14 ns.initday.off 15 ns.detrends.off15 # ns.detrends.off 16 16 ns.dqstats.off 17 17 ns.stacks.off … … 33 33 end 34 34 35 macro ns.detrends.on36 task ns.detrends.load37 active true38 end39 end40 35 41 36 macro ns.dqstats.on … … 46 41 47 42 macro ns.stacks.on 48 task ns.stacks.load49 active false50 end51 43 task ns.stacks.run 52 44 active true 53 45 end 54 task ns.stacks.confirm55 active false56 end57 46 end 58 47 59 48 macro ns.diffs.on 60 task ns.diffs.load61 active false62 end63 49 task ns.diffs.run 64 50 active true … … 72 58 end 73 59 74 macro ns.detrends.off75 task ns.detrends.load76 active false77 end78 end79 80 60 macro ns.dqstats.off 81 61 task ns.dqstats.load … … 85 65 86 66 macro ns.stacks.off 87 task ns.stacks.load88 active false89 end90 67 task ns.stacks.run 91 68 active false 92 69 end 93 task ns.stacks.confirm94 active false95 end96 97 70 end 98 71 99 72 macro ns.diffs.off 100 task ns.diffs.load101 active false102 end103 73 task ns.diffs.run 104 active false105 end106 end107 108 macro ns.stacks.confirm.on109 task ns.stacks.confirm110 active true111 end112 end113 114 macro ns.stacks.confirm.off115 task ns.stacks.confirm116 74 active false 117 75 end … … 128 86 book init nsDiffs 129 87 book init nsStacks 88 89 # Hard coded values 90 $camera_map:gpc1 = "GPC1" 91 $camera_map:gpc2 = "GPC2" 92 93 # Macro to manage the camera maps 94 macro ns.add.camera_map 95 if ($0 != 3) 96 echo "USAGE: ns.add.camera_map (dbname) (camera)" 97 break 98 end 99 100 $camera_map:$1 = $2 101 end 102 130 103 # 131 104 # Macros to control the book. 132 105 # 133 106 macro ns.add.date 134 if ($0 != 2)135 echo "USAGE: ns.add.date (YYYY-MM-DD) "107 if ($0 != 4) 108 echo "USAGE: ns.add.date (YYYY-MM-DD) (dbname) (camera)" 136 109 break 137 110 end 138 111 139 book newpage nsData $1 140 book setword nsData $1 nsState NEW 141 book setword nsData $1 dbname $DB:0 142 143 book newpage nsDiffs $1 144 book setword nsDiffs $1 nsDiffState TOWARP 145 book setword nsDiffs $1 nsObservingState UNKNOWN 146 147 book newpage nsStacks $1 148 book setword nsStacks $1 nsStackState TOWARP 112 if ($?camera_map:$2) 113 if ($camera_map:$2 != "$3") 114 echo "camera $3 does not match expected camera $camera_map:$2" 115 break 116 end 117 else 118 echo "dbname $2 does not have a camera_map entry" 119 break 120 end 121 122 # build key from date and database name 123 sprintf key "%s-%s" $1 $2 124 book newpage nsData $key 125 book setword nsData $key nsState NEW 126 book setword nsData $key date $1 127 book setword nsData $key dbname $2 128 book setword nsData $key camera $3 129 130 book newpage nsDiffs $key 131 book setword nsDiffs $key nsDiffState TOWARP 132 book setword nsDiffs $key nsObservingState UNKNOWN 133 134 book newpage nsStacks $key 135 book setword nsStacks $key nsStackState TOWARP 149 136 150 137 end … … 153 140 book npages nsData -var Npages 154 141 for i 0 $Npages 155 book getpage nsData $i -var date 156 book getword nsData $date nsState -var state 157 book getword nsStacks $date nsStackState -var Sstate 158 book getword nsDiffs $date nsDiffState -var Dstate 159 book getword nsDiffs $date nsObservingState -var Ostate 160 echo $date $state $Sstate $Dstate $Ostate 142 book getpage nsData $i -var key 143 book getword nsData $key date -var date 144 book getword nsData $key dbname -var dbname 145 book getword nsData $key camera -var camera 146 book getword nsData $key nsState -var state 147 book getword nsStacks $key nsStackState -var Sstate 148 book getword nsDiffs $key nsDiffState -var Dstate 149 book getword nsDiffs $key nsObservingState -var Ostate 150 # I've moved dbname to end as czartool reads information from here, and it may be fixed length 151 echo $key $date $state $Sstate $Dstate $Ostate $dbname $camera 152 161 153 end 162 154 end … … 179 171 periods -exec $LOADEXEC 180 172 periods -timeout 300 181 trange 0:00:00 1:00:00 -nmax 1173 trange 0:00:00 1:00:00 -nmax 2 182 174 npending 1 183 175 … … 186 178 stderr $LOGDIR/ns.initday.log 187 179 $today = `date -u +%Y-%m-%d` 188 book newpage nsData $today 189 book setword nsData $today nsState NEW 190 book newpage nsStacks $today 191 book setword nsStacks $today nsStackState TOWARP 192 book newpage nsDiffs $today 193 book setword nsDiffs $today nsDiffState TOWARP 194 book setword nsData $today dbname $DB:0 195 196 # command echo $today 197 $run = nightly_science.pl --clean_old --date $today --dbname $DB:0 198 # add_standard_args run 199 command $run 180 # loop over all databases 181 for i 0 $DB:n 182 $thisDB = $DB:$i 183 sprintf key "%s-%s" $today $thisDB 184 book newpage nsData $key 185 book setword nsData $key date $today 186 book setword nsData $key nsState NEW 187 book setword nsData $key dbname $thisDB 188 book setword nsData $key camera $camera_map:$thisDB 189 190 book newpage nsStacks $key 191 book setword nsStacks $key date $today 192 book setword nsStacks $key nsStackState TOWARP 193 book newpage nsDiffs $key 194 book setword nsDiffs $key date $today 195 book setword nsDiffs $key nsDiffState TOWARP 196 197 #command echo $today 198 $run = nightly_science.pl --clean_old --date $today --dbname $thisDB --camera $camera_map:$thisDB 199 command $run 200 end 200 201 end 201 202 … … 217 218 218 219 # 219 # Queue off any possible detrend verification runs220 #221 task ns.detrends.load222 host local223 periods -poll $LOADPOLL224 periods -exec $LOADEXEC225 periods -timeout 30226 trange 23:00:00 23:59:59 -nmax 1227 npending 1228 229 task.exec230 stdout $LOGDIR/ns.detrends.log231 stdout $LOGDIR/ns.detrends.log232 $today = `date -u +%Y-%m-%d`233 234 $run = nightly_science.pl --queue_detrends --date $today --dbname $DB:0235 # add_standard_args $run236 command $run237 end238 239 task.exit 0240 # nothign to do here241 end242 # locked list243 task.exit default244 showcommand failure245 end246 task.exit crash247 showcommand crash248 end249 # operation times out250 task.exit timeout251 showcommand timeout252 end253 end254 255 #256 220 # Queue dqstats runs 257 221 # 222 223 # CZW: 2015-05-05 One day we will stop making these, but that day is not today. 224 # According to Ken, we do not need to implement this for gpc2, as nothing is 225 # written on the OTIS/scheduler side to accept these products. We should 226 # continue to generate these for gpc1, as the machinery does exist to check 227 # for them, and not generating them may cause that to fail (even if nothing 228 # is done with the results). 229 258 230 task ns.dqstats.load 259 231 host local … … 270 242 $today = `date -u +%Y-%m-%d` 271 243 272 $run = nightly_science.pl --queue_dqstats --date $today --dbname $DB:0244 $run = nightly_science.pl --queue_dqstats --date $today --dbname gpc1 --camera GPC1 273 245 # add_standard_args $run 274 246 command $run … … 288 260 # operation times out 289 261 task.exit timeout 290 showcommand timeout291 end292 end293 294 #295 # Check to see if the warps are finished296 #297 task ns.stacks.load298 active false299 host local300 periods -poll $LOADPOLL301 periods -exec $LOADEXEC302 periods -timeout 120303 npending 1304 305 task.exec306 stdout NULL307 stderr $LOGDIR/ns.stacks.log308 309 book getpage nsData $ns_stackPAGE -var date310 book getword nsStacks $date nsStackState -var ns_STATE311 book getword nsDiffs $date nsDiffState -var ns_diff_STATE312 book getword nsData $date dbname -var DBNAME313 book npages nsData -var Npages314 315 if ($VERBOSE > 5)316 echo "ns.stacks.load: " $ns_stackPAGE $date $ns_STATE $Npages317 end318 319 $ns_stackPAGE ++320 if ($ns_stackPAGE >= $Npages) set ns_stackPAGE = 0321 option $date322 323 if ("$ns_STATE" != "TOWARP") break324 325 $run = nightly_science.pl --check_stacks --date $date --dbname $DBNAME326 # add_standard_args $run327 command $run328 329 end330 331 # success332 task.exit 0333 # convert 'stdout' to book format334 book delpage nsStacks $options:0335 ipptool2book stdout nsStacks -uniq -key date336 337 book getword nsStacks $options:0 nsNmacros -var ns_Nmacros338 if ("$ns_Nmacros" != "NULL")339 for i 0 $ns_Nmacros340 sprintf macroName "ns%dMacro" $i341 book getword nsStacks $options:0 $macroName -var macroCmd342 $macroCmd343 end344 end345 346 if ($VERBOSE > 2)347 book listbook nsData348 end349 end350 351 # locked list352 task.exit default353 showcommand failure354 end355 task.exit crash356 showcommand crash357 end358 #operation times out?359 task.exit timeout360 262 showcommand timeout 361 263 end … … 377 279 stderr $LOGDIR/ns.stacks.log 378 280 379 book getpage nsData $ns_RstackPAGE -var date 380 book getword nsStacks $date nsStackState -var ns_STATE 381 book getword nsDiffs $date nsDiffState -var ns_diff_STATE 382 book getword nsData $date dbname -var DBNAME 281 book getpage nsData $ns_RstackPAGE -var key 282 book getword nsStacks $key date -var date 283 book getword nsStacks $key nsStackState -var ns_STATE 284 book getword nsDiffs $key nsDiffState -var ns_diff_STATE 285 book getword nsData $key dbname -var DBNAME 286 book getword nsData $key camera -var CAMERA 383 287 book npages nsData -var Npages 384 288 385 289 if ($VERBOSE > 5) 386 echo "ns.stacks.run: " $ns_RstackPAGE $ date $ns_STATE $Npages290 echo "ns.stacks.run: " $ns_RstackPAGE $key $date $ns_STATE $Npages 387 291 end 388 292 389 293 $ns_RstackPAGE ++ 390 294 if ($ns_RstackPAGE >= $Npages) set ns_RstackPAGE = 0 391 option $ date295 option $key 392 296 393 297 if (("$ns_STATE" != "STACKING")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break 394 $run = nightly_science.pl --queue_stacks --date $date --dbname $DBNAME 298 $run = nightly_science.pl --queue_stacks --date $date --dbname $DBNAME --camera $CAMERA 395 299 # add_standard_args $run 396 300 command $run … … 430 334 end 431 335 432 #433 # Confirm that all the stacks that can be built have been built, or at least attempted.434 #435 task ns.stacks.confirm436 active false437 host local438 periods -poll $LOADPOLL439 periods -exec $LOADEXEC440 periods -timeout 120441 npending 1442 443 task.exec444 stdout NULL445 stderr $LOGDIR/ns.stacks.log446 447 book getpage nsData $ns_CstackPAGE -var date448 book getword nsStacks $date nsStackState -var ns_STATE449 book getword nsData $date dbname -var DBNAME450 book npages nsData -var Npages451 452 if ($VERBOSE > 5)453 echo "ns.stacks.confirm: " $ns_CstackPAGE $date $ns_STATE $Npages454 end455 456 $ns_CstackPAGE ++457 if ($ns_CstackPAGE >= $Npages) set ns_CstackPAGE = 0458 option $date459 460 if ("$ns_STATE" != "STACKING") break461 $run = nightly_science.pl --confirm_stacks --date $date --dbname $DBNAME462 # add_standard_args $run463 command $run464 end465 # success466 task.exit 0467 # convert 'stdout' to book format468 book delpage nsStacks $options:0469 ipptool2book stdout nsStacks -uniq -key date470 471 book getword nsStacks $options:0 nsNmacros -var ns_Nmacros472 if ("$ns_Nmacros" != "NULL")473 for i 0 $ns_Nmacros474 sprintf macroName "ns%dMacro" $i475 book getword nsStacks $options:0 $macroName -var macroCmd476 $macroCmd477 end478 end479 480 if ($VERBOSE > 2)481 book listbook nsData482 end483 end484 485 # locked list486 task.exit default487 showcommand failure488 end489 task.exit crash490 showcommand crash491 end492 #operation times out?493 task.exit timeout494 showcommand timeout495 end496 end497 498 499 #500 # Check to see if the warps are finished501 #502 task ns.diffs.load503 active false504 host local505 periods -poll $LOADPOLL506 periods -exec $LOADEXEC507 periods -timeout 120508 npending 1509 510 task.exec511 stdout NULL512 stderr $LOGDIR/ns.diffs.log513 514 book getpage nsData $ns_diffPAGE -var date515 book getword nsDiffs $date nsDiffState -var ns_STATE516 book getword nsStacks $date nsStackState -var ns_stack_STATE517 book getword nsData $date dbname -var DBNAME518 book npages nsData -var Npages519 520 if ($VERBOSE > 5)521 echo "ns.diffs.load: " $ns_diffPAGE $date $ns_STATE $Npages522 end523 524 $ns_diffPAGE ++525 if ($ns_diffPAGE >= $Npages) set ns_diffPAGE = 0526 option $date527 528 if ("$ns_STATE" != "TOWARP") break529 $run = nightly_science.pl --check_diffs --date $date --dbname $DBNAME530 # add_standard_args $run531 command $run532 end533 534 # success535 task.exit 0536 # convert 'stdout' to book format537 book delpage nsDiffs $options:0538 ipptool2book stdout nsDiffs -uniq -key date539 540 book getword nsDiffs $options:0 nsNmacros -var ns_Nmacros541 if ("$ns_Nmacros" != "NULL")542 for i 0 $ns_Nmacros543 sprintf macroName "ns%dMacro" $i544 book getword nsDiffs $options:0 $macroName -var macroCmd545 $macroCmd546 end547 end548 549 if ($VERBOSE > 2)550 book listbook nsData551 end552 end553 554 # locked list555 task.exit default556 showcommand failure557 end558 task.exit crash559 showcommand crash560 end561 #operation times out?562 task.exit timeout563 showcommand timeout564 end565 end566 336 567 337 # … … 580 350 stderr $LOGDIR/ns.diffs.log 581 351 582 book getpage nsData $ns_RdiffPAGE -var date 583 book getword nsDiffs $date nsDiffState -var ns_STATE 584 book getword nsStacks $date nsStackState -var ns_stack_STATE 585 book getword nsData $date dbname -var DBNAME 352 book getpage nsData $ns_RdiffPAGE -var key 353 book getword nsDiffs $key date -var date 354 book getword nsDiffs $key nsDiffState -var ns_STATE 355 book getword nsStacks $key nsStackState -var ns_stack_STATE 356 book getword nsData $key dbname -var DBNAME 357 book getword nsData $key camera -var CAMERA 586 358 book npages nsData -var Npages 587 359 … … 592 364 $ns_RdiffPAGE ++ 593 365 if ($ns_RdiffPAGE >= $Npages) set ns_RdiffPAGE = 0 594 option $ date366 option $key 595 367 596 368 if (("$ns_STATE" != "DIFFING")&&("$ns_STATE" != "QUEUEDIFFS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break 597 $run = nightly_science.pl --queue_diffs --date $date --dbname $DBNAME 369 $run = nightly_science.pl --queue_diffs --date $date --dbname $DBNAME --camera $CAMERA 598 370 # add_standard_args $run 599 371 command $run … … 635 407 636 408 409 # This is the graveyard of tasks that we've defined in the past, but either never use, or never let go active. 410 411 # if (0) 412 413 414 # macro ns.detrends.on 415 # task ns.detrends.load 416 # active true 417 # end 418 # end 419 420 421 # macro ns.detrends.off 422 # task ns.detrends.load 423 # active false 424 # end 425 # end 426 427 # macro ns.stacks.confirm.on 428 # task ns.stacks.confirm 429 # active true 430 # end 431 # end 432 433 # macro ns.stacks.confirm.off 434 # task ns.stacks.confirm 435 # active false 436 # end 437 # end 438 439 440 441 442 443 # # 444 # # Queue off any possible detrend verification runs 445 # # 446 # task ns.detrends.load 447 # host local 448 # periods -poll $LOADPOLL 449 # periods -exec $LOADEXEC 450 # periods -timeout 30 451 # trange 23:00:00 23:59:59 -nmax 1 452 # npending 1 453 454 # task.exec 455 # stdout $LOGDIR/ns.detrends.log 456 # stdout $LOGDIR/ns.detrends.log 457 # $today = `date -u +%Y-%m-%d` 458 459 # $run = nightly_science.pl --queue_detrends --date $today --dbname $DB:0 460 # # add_standard_args $run 461 # command $run 462 # end 463 464 # task.exit 0 465 # # nothign to do here 466 # end 467 # # locked list 468 # task.exit default 469 # showcommand failure 470 # end 471 # task.exit crash 472 # showcommand crash 473 # end 474 # # operation times out 475 # task.exit timeout 476 # showcommand timeout 477 # end 478 # end 479 480 481 # # 482 # # Check to see if the warps are finished 483 # # 484 # task ns.stacks.load 485 # active false 486 # host local 487 # periods -poll $LOADPOLL 488 # periods -exec $LOADEXEC 489 # periods -timeout 120 490 # npending 1 491 492 # task.exec 493 # stdout NULL 494 # stderr $LOGDIR/ns.stacks.log 495 496 # book getpage nsData $ns_stackPAGE -var key 497 # book getword nsStacks $key date -var date 498 # book getword nsStacks $key nsStackState -var ns_STATE 499 # book getword nsDiffs $key nsDiffState -var ns_diff_STATE 500 # book getword nsData $key dbname -var DBNAME 501 # book npages nsData -var Npages 502 503 # if ($VERBOSE > 5) 504 # echo "ns.stacks.load: " $ns_stackPAGE $date $DBNAME $ns_STATE $Npages 505 # end 506 507 # $ns_stackPAGE ++ 508 # if ($ns_stackPAGE >= $Npages) set ns_stackPAGE = 0 509 # option $key 510 511 # if ("$ns_STATE" != "TOWARP") break 512 513 # $run = nightly_science.pl --check_stacks --date $date --dbname $DBNAME 514 # # add_standard_args $run 515 # command $run 516 517 # end 518 519 # # success 520 # task.exit 0 521 # # convert 'stdout' to book format 522 # book delpage nsStacks $options:0 523 # ipptool2book stdout nsStacks -uniq -key date 524 525 # book getword nsStacks $options:0 nsNmacros -var ns_Nmacros 526 # if ("$ns_Nmacros" != "NULL") 527 # for i 0 $ns_Nmacros 528 # sprintf macroName "ns%dMacro" $i 529 # book getword nsStacks $options:0 $macroName -var macroCmd 530 # $macroCmd 531 # end 532 # end 533 534 # if ($VERBOSE > 2) 535 # book listbook nsData 536 # end 537 # end 538 539 # # locked list 540 # task.exit default 541 # showcommand failure 542 # end 543 # task.exit crash 544 # showcommand crash 545 # end 546 # #operation times out? 547 # task.exit timeout 548 # showcommand timeout 549 # end 550 # end 551 552 # # 553 # # Confirm that all the stacks that can be built have been built, or at least attempted. 554 # # 555 # task ns.stacks.confirm 556 # active false 557 # host local 558 # periods -poll $LOADPOLL 559 # periods -exec $LOADEXEC 560 # periods -timeout 120 561 # npending 1 562 563 # task.exec 564 # stdout NULL 565 # stderr $LOGDIR/ns.stacks.log 566 567 # book getpage nsData $ns_CstackPAGE -var key 568 # book getword nsStacks $key date -var date 569 # book getword nsStacks $key nsStackState -var ns_STATE 570 # book getword nsData $key dbname -var DBNAME 571 # book npages nsData -var Npages 572 573 # if ($VERBOSE > 5) 574 # echo "ns.stacks.confirm: " $ns_CstackPAGE $key $date $DBNAME $ns_STATE $Npages 575 # end 576 577 # $ns_CstackPAGE ++ 578 # if ($ns_CstackPAGE >= $Npages) set ns_CstackPAGE = 0 579 # option $key 580 581 # if ("$ns_STATE" != "STACKING") break 582 # $run = nightly_science.pl --confirm_stacks --date $date --dbname $DBNAME 583 # # add_standard_args $run 584 # command $run 585 # end 586 # # success 587 # task.exit 0 588 # # convert 'stdout' to book format 589 # book delpage nsStacks $options:0 590 # ipptool2book stdout nsStacks -uniq -key date 591 592 # book getword nsStacks $options:0 nsNmacros -var ns_Nmacros 593 # if ("$ns_Nmacros" != "NULL") 594 # for i 0 $ns_Nmacros 595 # sprintf macroName "ns%dMacro" $i 596 # book getword nsStacks $options:0 $macroName -var macroCmd 597 # $macroCmd 598 # end 599 # end 600 601 # if ($VERBOSE > 2) 602 # book listbook nsData 603 # end 604 # end 605 606 # # locked list 607 # task.exit default 608 # showcommand failure 609 # end 610 # task.exit crash 611 # showcommand crash 612 # end 613 # #operation times out? 614 # task.exit timeout 615 # showcommand timeout 616 # end 617 # end 618 619 620 621 622 # # 623 # # Check to see if the warps are finished 624 # # 625 # task ns.diffs.load 626 # active false 627 # host local 628 # periods -poll $LOADPOLL 629 # periods -exec $LOADEXEC 630 # periods -timeout 120 631 # npending 1 632 633 # task.exec 634 # stdout NULL 635 # stderr $LOGDIR/ns.diffs.log 636 637 # book getpage nsData $ns_diffPAGE -var key 638 # book getword nsDiffs $key date -var date 639 # book getword nsDiffs $key nsDiffState -var ns_STATE 640 # book getword nsStacks $key nsStackState -var ns_stack_STATE 641 # book getword nsData $key dbname -var DBNAME 642 # book npages nsData -var Npages 643 644 # if ($VERBOSE > 5) 645 # echo "ns.diffs.load: " $ns_diffPAGE $key $date $DBNAME $ns_STATE $Npages 646 # end 647 648 # $ns_diffPAGE ++ 649 # if ($ns_diffPAGE >= $Npages) set ns_diffPAGE = 0 650 # option $key 651 652 # if ("$ns_STATE" != "TOWARP") break 653 # $run = nightly_science.pl --check_diffs --date $date --dbname $DBNAME 654 # # add_standard_args $run 655 # command $run 656 # end 657 658 # # success 659 # task.exit 0 660 # # convert 'stdout' to book format 661 # book delpage nsDiffs $options:0 662 # ipptool2book stdout nsDiffs -uniq -key date 663 664 # book getword nsDiffs $options:0 nsNmacros -var ns_Nmacros 665 # if ("$ns_Nmacros" != "NULL") 666 # for i 0 $ns_Nmacros 667 # sprintf macroName "ns%dMacro" $i 668 # book getword nsDiffs $options:0 $macroName -var macroCmd 669 # $macroCmd 670 # end 671 # end 672 673 # if ($VERBOSE > 2) 674 # book listbook nsData 675 # end 676 # end 677 678 # # locked list 679 # task.exit default 680 # showcommand failure 681 # end 682 # task.exit crash 683 # showcommand crash 684 # end 685 # #operation times out? 686 # task.exit timeout 687 # showcommand timeout 688 # end 689 # end 690 691 692 # # END GRAVEYARD 693 # end 694 695 637 696 # # 638 697 # # Queue sweetspot runs
Note:
See TracChangeset
for help on using the changeset viewer.
