Changeset 27840 for branches/simtest_nebulous_branches/ippTasks/magic.pro
- Timestamp:
- May 3, 2010, 8:50:52 AM (16 years ago)
- Location:
- branches/simtest_nebulous_branches
- Files:
-
- 2 edited
-
. (modified) (1 prop)
-
ippTasks/magic.pro (modified) (13 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/simtest_nebulous_branches
- Property svn:mergeinfo changed
-
branches/simtest_nebulous_branches/ippTasks/magic.pro
r25021 r27840 10 10 book init magicToTree 11 11 book init magicToProcess 12 book init magicToDS13 book init magicDSToRevert14 12 15 13 ### Database lists 16 14 $magicToTree_DB = 0 17 15 $magicToProcess_DB = 0 18 $magicToDS_DB = 0 19 $magicDSToRevert_DB = 0 20 21 #list of stages 22 $STAGE:n = 0 23 list STAGE -add "raw" 24 list STAGE -add "chip" 25 list STAGE -add "camera" 26 list STAGE -add "warp" 27 list STAGE -add "diff" 28 $magicDSRevertStage = 0 16 $magicRevertTree_DB = 0 17 $magicRevertNode_DB = 0 29 18 30 19 ### Check status of tasks … … 32 21 book listbook magicToTree 33 22 book listbook magicToProcess 34 book listbook magicToDS35 book listbook magicDSToRevert36 23 end 37 24 … … 40 27 book init magicToTree 41 28 book init magicToProcess 42 book init magicToDS43 book init magicDSToRevert44 29 end 45 30 46 31 ### Turn tasks on 32 macro magic.revert.on 33 task magic.revert.node 34 active true 35 end 36 task magic.revert.tree 37 active true 38 end 39 end 40 47 41 macro magic.on 48 42 task magic.tree.load … … 58 52 active true 59 53 end 60 task magic.destreak.load 61 active true 62 end 63 task magic.destreak.run 64 active true 65 end 66 end 67 68 macro magic.ds.revert.on 69 task magic.ds.revert.load 70 active true 71 end 72 task magic.ds.revert.run 73 active true 74 end 75 end 54 magic.revert.on 55 end 56 57 macro magic.revert.off 58 task magic.revert.node 59 active false 60 end 61 task magic.revert.tree 62 active false 63 end 64 end 65 76 66 ### Turn tasks off 77 67 macro magic.off … … 88 78 active false 89 79 end 90 task magic.destreak.load 91 active false 92 end 93 task magic.destreak.run 94 active false 95 end 96 end 97 98 macro magic.ds.revert.off 99 task magic.ds.revert.load 100 active false 101 end 102 task magic.ds.revert.run 103 active false 104 end 105 end 106 macro magic.ds.off 107 task magic.destreak.load 108 active false 109 end 110 task magic.destreak.run 111 active false 112 end 113 end 114 macro magic.ds.on 115 task magic.destreak.load 116 active true 117 end 118 task magic.destreak.run 119 active true 120 end 121 end 122 123 macro magic.ds.status 124 echo magicToDS 125 book listbook magicToDS 126 echo "" 127 echo magicDSToRevert 128 book listbook magicDSToRevert 129 end 130 131 80 magic.revert.off 81 end 132 82 133 83 task magic.tree.load … … 262 212 263 213 periods -poll $LOADPOLL 264 periods -exec $LOADEXEC 265 periods -timeout 30 214 # periods -exec $LOADEXEC 215 periods -exec 30 216 periods -timeout 300 266 217 npending 1 267 218 … … 355 306 $WORKDIR = $WORKDIR_TEMPLATE 356 307 357 sprintf outroot "%s/%s/%s.mgc.%s.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID $NODE 358 359 ## generate output log based on filerule (convert the URI to a PATH) 360 ## caution with neb:// names here: 361 # $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot` 362 $logfile = $outroot.log 308 sprintf baseroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID 309 310 $logfile = $baseroot.$NODE.log 363 311 if ("$logfile" == "") 364 312 echo "WARNING: logfile not defined in magic.process.run" … … 371 319 mkdir $outpath 372 320 373 $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE -- outroot $outroot --logfile $logfile321 $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile 374 322 add_standard_args run 375 323 … … 401 349 end 402 350 403 task magic.destreak.load351 task magic.revert.node 404 352 host local 405 353 406 periods -poll $LOADPOLL 407 # this query can take a long time 408 periods -exec 10 409 periods -timeout 120 354 periods -poll 60.0 355 periods -exec 1800.0 356 periods -timeout 120.0 410 357 npending 1 411 358 412 359 stdout NULL 413 stderr $LOGDIR/magic.destreak.log 414 415 task.exec 416 $run = magicdstool -todestreak -limit 120 360 stderr $LOGSUBDIR/revertnode.log 361 362 task.exec 363 if ($LABEL:n == 0) break 364 $run = magictool -revertnode 417 365 if ($DB:n == 0) 418 366 option DEFAULT 419 367 else 420 368 # save the DB name for the exit tasks 421 option $DB:$magicToDS_DB 422 $run = $run -dbname $DB:$magicToDS_DB 423 $magicToDS_DB ++ 424 if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0 425 end 426 add_poll_args run 369 option $DB:$magicRevertNode_DB 370 $run = $run -dbname $DB:$magicRevertNode_DB 371 $magicRevertNode_DB ++ 372 if ($magicRevertNode_DB >= $DB:n) set magicRevertNode_DB = 0 373 end 427 374 add_poll_labels run 428 375 command $run … … 431 378 # success 432 379 task.exit 0 433 # convert 'stdout' to book format434 ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT435 if ($VERBOSE > 2)436 book listbook magicToDS437 end438 439 # delete existing entries in the appropriate pantaskStates440 process_cleanup magicToDS441 380 end 442 381 … … 455 394 end 456 395 end 457 458 task magic.destreak.run 459 periods -poll $RUNPOLL 460 periods -exec $RUNEXEC 461 periods -timeout 60 462 463 task.exec 464 book npages magicToDS -var N 465 if ($N == 0) break 466 if ($NETWORK == 0) break 467 468 # look for new images (pantaskState == INIT) 469 book getpage magicToDS 0 -var pageName -key pantaskState INIT 470 if ("$pageName" == "NULL") break 471 472 book setword magicToDS $pageName pantaskState RUN 473 book getword magicToDS $pageName exp_id -var EXP_ID 474 book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID 475 book getword magicToDS $pageName camera -var CAMERA 476 book getword magicToDS $pageName streaks_uri -var STREAKS 477 book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS 478 book getword magicToDS $pageName stage -var STAGE 479 book getword magicToDS $pageName stage_id -var STAGE_ID 480 book getword magicToDS $pageName component -var COMPONENT 481 book getword magicToDS $pageName uri -var URI 482 book getword magicToDS $pageName path_base -var PATH_BASE 483 book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE 484 book getword magicToDS $pageName outroot -var OUTROOT 485 book getword magicToDS $pageName recoveryroot -var RECROOT 486 book getword magicToDS $pageName re_place -var REPLACE 487 book getword magicToDS $pageName dbname -var DBNAME 488 489 sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT 490 491 substr $COMPONENT 0 3 COMP_HEAD 492 if ("$COMP_HEAD" == "sky") 493 set.host.for.skycell $COMPONENT 494 else 495 # assume component is a class_id, if not we will default to anyhost 496 set.host.for.camera $CAMERA $COMPONENT 497 end 498 499 # TODO: do not add recoveryroot or replace if they are null or zero 500 501 $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE 502 503 add_standard_args run 504 505 # save the pageName for future reference below 506 options $pageName 507 508 # create the command line 509 if ($VERBOSE > 1) 510 echo command $run 511 end 512 command $run 513 end 514 515 # default exit status 516 task.exit 0 517 process_exit magicToDS $options:0 $JOB_STATUS 518 end 519 520 # locked list 521 task.exit default 522 showcommand failure 523 process_exit magicToDS $options:0 $JOB_STATUS 524 end 525 526 task.exit crash 527 showcommand crash 528 book setword magicToDS $options:0 pantaskState CRASH 529 end 530 531 # operation timed out? 532 task.exit timeout 533 showcommand timeout 534 book setword magicToDS $options:0 pantaskState TIMEOUT 535 end 536 end 537 538 task magic.ds.revert.load 396 task magic.revert.tree 539 397 host local 540 398 541 periods -poll $LOADPOLL542 periods -exec $LOADEXEC543 periods -timeout 20399 periods -poll 60.0 400 periods -exec 1800.0 401 periods -timeout 120.0 544 402 npending 1 545 active false546 403 547 404 stdout NULL 548 stderr $LOGDIR/magic.ds.revert.log 549 550 task.exec 551 $run = magicdstool -torevert -stage $STAGE:$magicDSRevertStage 552 $magicDSRevertStage ++ 553 if ($magicDSRevertStage >= $STAGE:n) set magicDSRevertStage = 0 554 405 stderr $LOGSUBDIR/reverttree.log 406 407 task.exec 408 if ($LABEL:n == 0) break 409 $run = magictool -reverttree 555 410 if ($DB:n == 0) 556 411 option DEFAULT 557 412 else 558 559 413 # save the DB name for the exit tasks 560 option $DB:$magicDSToRevert_DB 561 $run = $run -dbname $DB:$magicDSToRevert_DB 562 563 # only bump database number after we have gone through all of the stages 564 if ($magicDSRevertStage == 0) 565 $magicDSToRevert_DB ++ 566 end 567 if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0 568 end 569 add_poll_args run 414 option $DB:$magicRevertTree_DB 415 $run = $run -dbname $DB:$magicRevertTree_DB 416 $magicRevertTree_DB ++ 417 if ($magicRevertTree_DB >= $DB:n) set magicRevertTree_DB = 0 418 end 570 419 add_poll_labels run 571 420 command $run … … 574 423 # success 575 424 task.exit 0 576 # convert 'stdout' to book format577 ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT578 if ($VERBOSE > 2)579 book listbook magicDSToRevert580 end581 582 # delete existing entries in the appropriate pantaskStates583 process_cleanup magicDSToRevert584 425 end 585 426 … … 598 439 end 599 440 end 600 601 task magic.ds.revert.run602 periods -poll $RUNPOLL603 periods -exec $RUNEXEC604 periods -timeout 60605 active false606 607 task.exec608 book npages magicDSToRevert -var N609 if ($N == 0) break610 if ($NETWORK == 0) break611 612 # look for new images (pantaskState == INIT)613 book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT614 if ("$pageName" == "NULL") break615 616 book setword magicDSToRevert $pageName pantaskState RUN617 book getword magicDSToRevert $pageName exp_id -var EXP_ID618 book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID619 book getword magicDSToRevert $pageName camera -var CAMERA620 book getword magicDSToRevert $pageName stage -var STAGE621 book getword magicDSToRevert $pageName stage_id -var STAGE_ID622 book getword magicDSToRevert $pageName component -var COMPONENT623 book getword magicDSToRevert $pageName path_base -var PATH_BASE624 book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE625 book getword magicDSToRevert $pageName outroot -var OUTROOT626 book getword magciDSToRevert $pageName bytes -var BYTES627 book getword magciDSToRevert $pageName md5sum -var md5sum628 # book getword magicDSToRevert $pageName recoveryroot -var RECROOT629 book getword magicDSToRevert $pageName re_place -var REPLACE630 book getword magicDSToRevert $pageName dbname -var DBNAME631 632 sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT633 634 substr $COMPONENT 0 3 COMP_HEAD635 if ("$COMP_HEAD" == "sky")636 set.host.for.skycell $COMPONENT637 else638 # assume component is a class_id, if not we will default to anyhost639 set.host.for.camera $CAMERA $COMPONENT640 end641 642 $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE643 644 add_standard_args run645 646 # save the pageName for future reference below647 options $pageName648 649 # create the command line650 if ($VERBOSE > 1)651 echo command $run652 end653 command $run654 end655 656 # default exit status657 task.exit 0658 process_exit magicDSToRevert $options:0 $JOB_STATUS659 end660 661 # locked list662 task.exit default663 showcommand failure664 process_exit magicDSToRevert $options:0 $JOB_STATUS665 end666 667 task.exit crash668 showcommand crash669 book setword magicDSToRevert $options:0 pantaskState CRASH670 end671 672 # operation timed out?673 task.exit timeout674 showcommand timeout675 book setword magicDSToRevert $options:0 pantaskState TIMEOUT676 end677 end678
Note:
See TracChangeset
for help on using the changeset viewer.
