Index: trunk/ippTasks/Makefile.am =================================================================== --- trunk/ippTasks/Makefile.am (revision 25418) +++ trunk/ippTasks/Makefile.am (revision 25419) @@ -17,4 +17,5 @@ warp.pro \ magic.pro \ + destreak.pro \ diff.pro \ stack.pro \ Index: trunk/ippTasks/destreak.pro =================================================================== --- trunk/ippTasks/destreak.pro (revision 25419) +++ trunk/ippTasks/destreak.pro (revision 25419) @@ -0,0 +1,416 @@ +## destreak.pro : support for the streak removal : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/destreak +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init magicToDS +book init magicDSToRevert + +### Database lists +$magicToDS_DB = 0 +$magicDSToRevert_DB = 0 + +#list of stages +$STAGE:n = 0 +list STAGE -add "raw" +list STAGE -add "chip" +list STAGE -add "camera" +list STAGE -add "warp" +list STAGE -add "diff" + +$magicDSStage = 0 +$magicDSRevertStage = 0 + +### Check status of tasks +macro magic.status + book listbook magicToDS + book listbook magicDSToRevert +end + +### Reset tasks +macro magic.reset + book init magicToDS + book init magicDSToRevert +end + +### Turn tasks on +macro destreak.on + task destreak.load + active true + end + task destreak.run + active true + end +end + +macro destreak.revert.on + task destreak.revert.load + active true + end + task destreak.revert.run + active true + end +end +### Turn tasks off +macro destreak.off + task destreak.load + active false + end + task destreak.run + active false + end +end + +macro destreak.revert.off + task destreak.revert.load + active false + end + task destreak.revert.run + active false + end +end + +macro destreak.status + echo magicToDS + book listbook magicToDS + echo "" + echo magicDSToRevert + book listbook magicDSToRevert +end + +task destreak.load + host local + + periods -poll $LOADPOLL + # this query can take a long time (XXX: the long time should be fixed now) + periods -exec 10 + periods -timeout 120 + npending 1 + + stdout NULL + stderr $LOGDIR/magic.destreak.log + + task.exec + $run = magicdstool -todestreak -limit 120 -stage $STAGE:$magicDSStage + $magicDSStage ++ + if ($magicDSStage >= $STAGE:n) set magicDSStage = 0 + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicToDS_DB + $run = $run -dbname $DB:$magicToDS_DB + $magicToDS_DB ++ + if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToDS + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToDS + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages magicToDS -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicToDS 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicToDS $pageName pantaskState RUN + book getword magicToDS $pageName exp_id -var EXP_ID + book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicToDS $pageName camera -var CAMERA + book getword magicToDS $pageName streaks_uri -var STREAKS + book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS + book getword magicToDS $pageName stage -var STAGE + book getword magicToDS $pageName stage_id -var STAGE_ID + book getword magicToDS $pageName component -var COMPONENT + book getword magicToDS $pageName uri -var URI + book getword magicToDS $pageName path_base -var PATH_BASE + book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE + book getword magicToDS $pageName outroot -var OUTROOT + book getword magicToDS $pageName recoveryroot -var RECROOT + book getword magicToDS $pageName re_place -var REPLACE + book getword magicToDS $pageName dbname -var DBNAME + + sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT + + substr $COMPONENT 0 3 COMP_HEAD + if ("$COMP_HEAD" == "sky") + set.host.for.skycell $COMPONENT + else + # assume component is a class_id, if not we will default to anyhost + set.host.for.camera $CAMERA $COMPONENT + end + + # TODO: do not add recoveryroot or replace if they are null or zero + + $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit 0 + process_exit magicToDS $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicToDS $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicToDS $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicToDS $options:0 pantaskState TIMEOUT + end +end + +task destreak.revert.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 20 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/destreak.revert.log + + task.exec + $run = magicdstool -torevert -stage $STAGE:$magicDSRevertStage + $magicDSRevertStage ++ + if ($magicDSRevertStage >= $STAGE:n) set magicDSRevertStage = 0 + + if ($DB:n == 0) + option DEFAULT + else + + # save the DB name for the exit tasks + option $DB:$magicDSToRevert_DB + $run = $run -dbname $DB:$magicDSToRevert_DB + + # only bump database number after we have gone through all of the stages + if ($magicDSRevertStage == 0) + $magicDSToRevert_DB ++ + end + if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicDSToRevert + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicDSToRevert + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.revert.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages magicDSToRevert -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicDSToRevert $pageName pantaskState RUN + book getword magicDSToRevert $pageName exp_id -var EXP_ID + book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicDSToRevert $pageName camera -var CAMERA + book getword magicDSToRevert $pageName stage -var STAGE + book getword magicDSToRevert $pageName stage_id -var STAGE_ID + book getword magicDSToRevert $pageName component -var COMPONENT + book getword magicDSToRevert $pageName path_base -var PATH_BASE + book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE + book getword magicDSToRevert $pageName outroot -var OUTROOT + book getword magciDSToRevert $pageName bytes -var BYTES + book getword magciDSToRevert $pageName md5sum -var md5sum +# book getword magicDSToRevert $pageName recoveryroot -var RECROOT + book getword magicDSToRevert $pageName re_place -var REPLACE + book getword magicDSToRevert $pageName dbname -var DBNAME + + sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT + + substr $COMPONENT 0 3 COMP_HEAD + if ("$COMP_HEAD" == "sky") + set.host.for.skycell $COMPONENT + else + # assume component is a class_id, if not we will default to anyhost + set.host.for.camera $CAMERA $COMPONENT + end + + $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit 0 + process_exit magicDSToRevert $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicDSToRevert $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicDSToRevert $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicDSToRevert $options:0 pantaskState TIMEOUT + end +end + +task destreak.completed.revert + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 20 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/destreak.completed.revert + + task.exec + $run = magicdstool -completedrevert + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicDSToRevert_DB + $run = $run -dbname $DB:$magicDSToRevert_DB + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicDSToRevert + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicDSToRevert + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: trunk/ippTasks/magic.pro =================================================================== --- trunk/ippTasks/magic.pro (revision 25418) +++ trunk/ippTasks/magic.pro (revision 25419) @@ -10,21 +10,8 @@ book init magicToTree book init magicToProcess -book init magicToDS -book init magicDSToRevert ### Database lists $magicToTree_DB = 0 $magicToProcess_DB = 0 -$magicToDS_DB = 0 -$magicDSToRevert_DB = 0 - -#list of stages -$STAGE:n = 0 -list STAGE -add "raw" -list STAGE -add "chip" -list STAGE -add "camera" -list STAGE -add "warp" -list STAGE -add "diff" -$magicDSRevertStage = 0 ### Check status of tasks @@ -32,6 +19,4 @@ book listbook magicToTree book listbook magicToProcess - book listbook magicToDS - book listbook magicDSToRevert end @@ -40,6 +25,4 @@ book init magicToTree book init magicToProcess - book init magicToDS - book init magicDSToRevert end @@ -58,20 +41,6 @@ active true end - task magic.destreak.load - active true - end - task magic.destreak.run - active true - end -end - -macro magic.ds.revert.on - task magic.ds.revert.load - active true - end - task magic.ds.revert.run - active true - end -end +end + ### Turn tasks off macro magic.off @@ -88,46 +57,5 @@ active false end - task magic.destreak.load - active false - end - task magic.destreak.run - active false - end -end - -macro magic.ds.revert.off - task magic.ds.revert.load - active false - end - task magic.ds.revert.run - active false - end -end -macro magic.ds.off - task magic.destreak.load - active false - end - task magic.destreak.run - active false - end -end -macro magic.ds.on - task magic.destreak.load - active true - end - task magic.destreak.run - active true - end -end - -macro magic.ds.status - echo magicToDS - book listbook magicToDS - echo "" - echo magicDSToRevert - book listbook magicDSToRevert -end - - +end task magic.tree.load @@ -400,279 +328,2 @@ end end - -task magic.destreak.load - host local - - periods -poll $LOADPOLL - # this query can take a long time - periods -exec 10 - periods -timeout 120 - npending 1 - - stdout NULL - stderr $LOGDIR/magic.destreak.log - - task.exec - $run = magicdstool -todestreak -limit 120 - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$magicToDS_DB - $run = $run -dbname $DB:$magicToDS_DB - $magicToDS_DB ++ - if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0 - end - add_poll_args run - add_poll_labels run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook magicToDS - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup magicToDS - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task magic.destreak.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - - task.exec - book npages magicToDS -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images (pantaskState == INIT) - book getpage magicToDS 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword magicToDS $pageName pantaskState RUN - book getword magicToDS $pageName exp_id -var EXP_ID - book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID - book getword magicToDS $pageName camera -var CAMERA - book getword magicToDS $pageName streaks_uri -var STREAKS - book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS - book getword magicToDS $pageName stage -var STAGE - book getword magicToDS $pageName stage_id -var STAGE_ID - book getword magicToDS $pageName component -var COMPONENT - book getword magicToDS $pageName uri -var URI - book getword magicToDS $pageName path_base -var PATH_BASE - book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE - book getword magicToDS $pageName outroot -var OUTROOT - book getword magicToDS $pageName recoveryroot -var RECROOT - book getword magicToDS $pageName re_place -var REPLACE - book getword magicToDS $pageName dbname -var DBNAME - - sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT - - substr $COMPONENT 0 3 COMP_HEAD - if ("$COMP_HEAD" == "sky") - set.host.for.skycell $COMPONENT - else - # assume component is a class_id, if not we will default to anyhost - set.host.for.camera $CAMERA $COMPONENT - end - - # TODO: do not add recoveryroot or replace if they are null or zero - - $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE - - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit 0 - process_exit magicToDS $options:0 $JOB_STATUS - end - - # locked list - task.exit default - showcommand failure - process_exit magicToDS $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword magicToDS $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword magicToDS $options:0 pantaskState TIMEOUT - end -end - -task magic.ds.revert.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 20 - npending 1 - active false - - stdout NULL - stderr $LOGDIR/magic.ds.revert.log - - task.exec - $run = magicdstool -torevert -stage $STAGE:$magicDSRevertStage - $magicDSRevertStage ++ - if ($magicDSRevertStage >= $STAGE:n) set magicDSRevertStage = 0 - - if ($DB:n == 0) - option DEFAULT - else - - # save the DB name for the exit tasks - option $DB:$magicDSToRevert_DB - $run = $run -dbname $DB:$magicDSToRevert_DB - - # only bump database number after we have gone through all of the stages - if ($magicDSRevertStage == 0) - $magicDSToRevert_DB ++ - end - if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0 - end - add_poll_args run - add_poll_labels run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook magicDSToRevert - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup magicDSToRevert - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task magic.ds.revert.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active false - - task.exec - book npages magicDSToRevert -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images (pantaskState == INIT) - book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword magicDSToRevert $pageName pantaskState RUN - book getword magicDSToRevert $pageName exp_id -var EXP_ID - book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID - book getword magicDSToRevert $pageName camera -var CAMERA - book getword magicDSToRevert $pageName stage -var STAGE - book getword magicDSToRevert $pageName stage_id -var STAGE_ID - book getword magicDSToRevert $pageName component -var COMPONENT - book getword magicDSToRevert $pageName path_base -var PATH_BASE - book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE - book getword magicDSToRevert $pageName outroot -var OUTROOT - book getword magciDSToRevert $pageName bytes -var BYTES - book getword magciDSToRevert $pageName md5sum -var md5sum -# book getword magicDSToRevert $pageName recoveryroot -var RECROOT - book getword magicDSToRevert $pageName re_place -var REPLACE - book getword magicDSToRevert $pageName dbname -var DBNAME - - sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT - - substr $COMPONENT 0 3 COMP_HEAD - if ("$COMP_HEAD" == "sky") - set.host.for.skycell $COMPONENT - else - # assume component is a class_id, if not we will default to anyhost - set.host.for.camera $CAMERA $COMPONENT - end - - $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE - - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit 0 - process_exit magicDSToRevert $options:0 $JOB_STATUS - end - - # locked list - task.exit default - showcommand failure - process_exit magicDSToRevert $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword magicDSToRevert $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword magicDSToRevert $options:0 pantaskState TIMEOUT - end -end -