Index: branches/simtest_nebulous_branches/ippTasks/magic.pro =================================================================== --- branches/simtest_nebulous_branches/ippTasks/magic.pro (revision 25061) +++ branches/simtest_nebulous_branches/ippTasks/magic.pro (revision 27840) @@ -10,21 +10,10 @@ book init magicToTree book init magicToProcess -book init magicToDS -book init magicDSToRevert ### Database lists $magicToTree_DB = 0 $magicToProcess_DB = 0 -$magicToDS_DB = 0 -$magicDSToRevert_DB = 0 - -#list of stages -$STAGE:n = 0 -list STAGE -add "raw" -list STAGE -add "chip" -list STAGE -add "camera" -list STAGE -add "warp" -list STAGE -add "diff" -$magicDSRevertStage = 0 +$magicRevertTree_DB = 0 +$magicRevertNode_DB = 0 ### Check status of tasks @@ -32,6 +21,4 @@ book listbook magicToTree book listbook magicToProcess - book listbook magicToDS - book listbook magicDSToRevert end @@ -40,9 +27,16 @@ book init magicToTree book init magicToProcess - book init magicToDS - book init magicDSToRevert end ### Turn tasks on +macro magic.revert.on + task magic.revert.node + active true + end + task magic.revert.tree + active true + end +end + macro magic.on task magic.tree.load @@ -58,20 +52,16 @@ active true end - task magic.destreak.load - active true - end - task magic.destreak.run - active true - end -end - -macro magic.ds.revert.on - task magic.ds.revert.load - active true - end - task magic.ds.revert.run - active true - end -end + magic.revert.on +end + +macro magic.revert.off + task magic.revert.node + active false + end + task magic.revert.tree + active false + end +end + ### Turn tasks off macro magic.off @@ -88,46 +78,6 @@ active false end - task magic.destreak.load - active false - end - task magic.destreak.run - active false - end -end - -macro magic.ds.revert.off - task magic.ds.revert.load - active false - end - task magic.ds.revert.run - active false - end -end -macro magic.ds.off - task magic.destreak.load - active false - end - task magic.destreak.run - active false - end -end -macro magic.ds.on - task magic.destreak.load - active true - end - task magic.destreak.run - active true - end -end - -macro magic.ds.status - echo magicToDS - book listbook magicToDS - echo "" - echo magicDSToRevert - book listbook magicDSToRevert -end - - + magic.revert.off +end task magic.tree.load @@ -262,6 +212,7 @@ periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 300 npending 1 @@ -355,10 +306,7 @@ $WORKDIR = $WORKDIR_TEMPLATE - sprintf outroot "%s/%s/%s.mgc.%s.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID $NODE - - ## generate output log based on filerule (convert the URI to a PATH) - ## caution with neb:// names here: - # $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot` - $logfile = $outroot.log + sprintf baseroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID + + $logfile = $baseroot.$NODE.log if ("$logfile" == "") echo "WARNING: logfile not defined in magic.process.run" @@ -371,5 +319,5 @@ mkdir $outpath - $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --outroot $outroot --logfile $logfile + $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile add_standard_args run @@ -401,28 +349,27 @@ end -task magic.destreak.load +task magic.revert.node host local - periods -poll $LOADPOLL - # this query can take a long time - periods -exec 10 - periods -timeout 120 + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 npending 1 stdout NULL - stderr $LOGDIR/magic.destreak.log - - task.exec - $run = magicdstool -todestreak -limit 120 + stderr $LOGSUBDIR/revertnode.log + + task.exec + if ($LABEL:n == 0) break + $run = magictool -revertnode if ($DB:n == 0) option DEFAULT else # save the DB name for the exit tasks - option $DB:$magicToDS_DB - $run = $run -dbname $DB:$magicToDS_DB - $magicToDS_DB ++ - if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0 - end - add_poll_args run + option $DB:$magicRevertNode_DB + $run = $run -dbname $DB:$magicRevertNode_DB + $magicRevertNode_DB ++ + if ($magicRevertNode_DB >= $DB:n) set magicRevertNode_DB = 0 + end add_poll_labels run command $run @@ -431,12 +378,4 @@ # success task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook magicToDS - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup magicToDS end @@ -455,117 +394,27 @@ end end - -task magic.destreak.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - - task.exec - book npages magicToDS -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images (pantaskState == INIT) - book getpage magicToDS 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword magicToDS $pageName pantaskState RUN - book getword magicToDS $pageName exp_id -var EXP_ID - book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID - book getword magicToDS $pageName camera -var CAMERA - book getword magicToDS $pageName streaks_uri -var STREAKS - book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS - book getword magicToDS $pageName stage -var STAGE - book getword magicToDS $pageName stage_id -var STAGE_ID - book getword magicToDS $pageName component -var COMPONENT - book getword magicToDS $pageName uri -var URI - book getword magicToDS $pageName path_base -var PATH_BASE - book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE - book getword magicToDS $pageName outroot -var OUTROOT - book getword magicToDS $pageName recoveryroot -var RECROOT - book getword magicToDS $pageName re_place -var REPLACE - book getword magicToDS $pageName dbname -var DBNAME - - sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT - - substr $COMPONENT 0 3 COMP_HEAD - if ("$COMP_HEAD" == "sky") - set.host.for.skycell $COMPONENT - else - # assume component is a class_id, if not we will default to anyhost - set.host.for.camera $CAMERA $COMPONENT - end - - # TODO: do not add recoveryroot or replace if they are null or zero - - $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE - - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit 0 - process_exit magicToDS $options:0 $JOB_STATUS - end - - # locked list - task.exit default - showcommand failure - process_exit magicToDS $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword magicToDS $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword magicToDS $options:0 pantaskState TIMEOUT - end -end - -task magic.ds.revert.load +task magic.revert.tree host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 20 + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 npending 1 - active false stdout NULL - stderr $LOGDIR/magic.ds.revert.log - - task.exec - $run = magicdstool -torevert -stage $STAGE:$magicDSRevertStage - $magicDSRevertStage ++ - if ($magicDSRevertStage >= $STAGE:n) set magicDSRevertStage = 0 - + stderr $LOGSUBDIR/reverttree.log + + task.exec + if ($LABEL:n == 0) break + $run = magictool -reverttree if ($DB:n == 0) option DEFAULT else - # save the DB name for the exit tasks - option $DB:$magicDSToRevert_DB - $run = $run -dbname $DB:$magicDSToRevert_DB - - # only bump database number after we have gone through all of the stages - if ($magicDSRevertStage == 0) - $magicDSToRevert_DB ++ - end - if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0 - end - add_poll_args run + option $DB:$magicRevertTree_DB + $run = $run -dbname $DB:$magicRevertTree_DB + $magicRevertTree_DB ++ + if ($magicRevertTree_DB >= $DB:n) set magicRevertTree_DB = 0 + end add_poll_labels run command $run @@ -574,12 +423,4 @@ # success task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook magicDSToRevert - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup magicDSToRevert end @@ -598,81 +439,2 @@ end end - -task magic.ds.revert.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active false - - task.exec - book npages magicDSToRevert -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images (pantaskState == INIT) - book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword magicDSToRevert $pageName pantaskState RUN - book getword magicDSToRevert $pageName exp_id -var EXP_ID - book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID - book getword magicDSToRevert $pageName camera -var CAMERA - book getword magicDSToRevert $pageName stage -var STAGE - book getword magicDSToRevert $pageName stage_id -var STAGE_ID - book getword magicDSToRevert $pageName component -var COMPONENT - book getword magicDSToRevert $pageName path_base -var PATH_BASE - book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE - book getword magicDSToRevert $pageName outroot -var OUTROOT - book getword magciDSToRevert $pageName bytes -var BYTES - book getword magciDSToRevert $pageName md5sum -var md5sum -# book getword magicDSToRevert $pageName recoveryroot -var RECROOT - book getword magicDSToRevert $pageName re_place -var REPLACE - book getword magicDSToRevert $pageName dbname -var DBNAME - - sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT - - substr $COMPONENT 0 3 COMP_HEAD - if ("$COMP_HEAD" == "sky") - set.host.for.skycell $COMPONENT - else - # assume component is a class_id, if not we will default to anyhost - set.host.for.camera $CAMERA $COMPONENT - end - - $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE - - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit 0 - process_exit magicDSToRevert $options:0 $JOB_STATUS - end - - # locked list - task.exit default - showcommand failure - process_exit magicDSToRevert $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword magicDSToRevert $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword magicDSToRevert $options:0 pantaskState TIMEOUT - end -end -