Index: /branches/eam_branches/20090715/ippTasks/Makefile.am =================================================================== --- /branches/eam_branches/20090715/ippTasks/Makefile.am (revision 25624) +++ /branches/eam_branches/20090715/ippTasks/Makefile.am (revision 25625) @@ -17,4 +17,5 @@ warp.pro \ magic.pro \ + destreak.pro \ diff.pro \ stack.pro \ Index: /branches/eam_branches/20090715/ippTasks/destreak.pro =================================================================== --- /branches/eam_branches/20090715/ippTasks/destreak.pro (revision 25625) +++ /branches/eam_branches/20090715/ippTasks/destreak.pro (revision 25625) @@ -0,0 +1,485 @@ +## destreak.pro : support for the streak removal : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/destreak +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init magicToDS +book init magicDSToRevert + +### indexes into Database lists +$magicToDS_DB = 0 +$magicDSAdvance_DB = 0 +$magicDSToRevert_DB = 0 +$magicDSCompletedRevert_DB = 0 + +#list of stages +$DS_STAGE:n = 0 +list DS_STAGE -add "raw" +list DS_STAGE -add "chip" +list DS_STAGE -add "camera" +list DS_STAGE -add "warp" +list DS_STAGE -add "diff" + +$magicDSStage = 0 +$magicDSRevertStage = 0 + +### Check status of tasks +macro magic.status + book listbook magicToDS + book listbook magicDSToRevert +end + +### Reset tasks +macro magic.reset + book init magicToDS + book init magicDSToRevert +end + +### Turn tasks on +macro destreak.on + # destreak and revert should not run at the same time + destreak.revert.off + + task destreak.load + active true + end + task destreak.run + active true + end + task destreak.advance + active true + end +end + +macro destreak.revert.on + # destreak and revert should not run at the same time + destreak.off + + task destreak.revert.load + active true + end + task destreak.revert.run + active true + end + task destreak.completed.revert + active true + end +end + +### Turn tasks off +macro destreak.off + task destreak.load + active false + end + task destreak.run + active false + end + task destreak.advance + active false + end +end + +macro destreak.revert.off + task destreak.revert.load + active false + end + task destreak.revert.run + active false + end + task destreak.completed.revert + active false + end +end + +macro destreak.status + echo magicToDS + book listbook magicToDS + echo "" + echo magicDSToRevert + book listbook magicDSToRevert +end + +task destreak.load + host local + + periods -poll $LOADPOLL + # this query can take a long time (XXX: the long time should be fixed now) + periods -exec 10 + periods -timeout 120 + npending 1 + +# stdout NULL +# stderr $LOGSUBDIR/destreak.load.log + + task.exec + $run = magicdstool -todestreak -limit 120 -stage $DS_STAGE:$magicDSStage + $magicDSStage ++ + if ($magicDSStage >= $DS_STAGE:n) set magicDSStage = 0 + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicToDS_DB + $run = $run -dbname $DB:$magicToDS_DB + + # only bump database number after we have gone through all of the stages + if ($magicDSStage == 0) + $magicToDS_DB ++ + if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0 + end + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToDS + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToDS + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages magicToDS -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicToDS 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicToDS $pageName pantaskState RUN + book getword magicToDS $pageName exp_id -var EXP_ID + book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicToDS $pageName camera -var CAMERA + book getword magicToDS $pageName streaks_uri -var STREAKS + book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS + book getword magicToDS $pageName stage -var STAGE + book getword magicToDS $pageName stage_id -var STAGE_ID + book getword magicToDS $pageName component -var COMPONENT + book getword magicToDS $pageName uri -var URI + book getword magicToDS $pageName path_base -var PATH_BASE + book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE + book getword magicToDS $pageName outroot -var OUTROOT + book getword magicToDS $pageName recoveryroot -var RECROOT + book getword magicToDS $pageName re_place -var REPLACE + book getword magicToDS $pageName dbname -var DBNAME + + sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT + + substr $COMPONENT 0 3 COMP_HEAD + if ("$COMP_HEAD" == "sky") + set.host.for.skycell $COMPONENT + else + # assume component is a class_id, if not we will default to anyhost + set.host.for.camera $CAMERA $COMPONENT + end + + # TODO: do not add recoveryroot or replace if they are null or zero + + $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit 0 + process_exit magicToDS $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicToDS $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicToDS $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicToDS $options:0 pantaskState TIMEOUT + end +end + +task destreak.advance + # task to finish processing for magicDSRuns + host local + + periods -poll $LOADPOLL + #periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 20 + npending 1 + +# stdout NULL +# stderr $LOGSUBDIR/destreak.advance.log + + task.exec + $run = magicdstool -advancerun + if ($DB:n != 0) + + $run = $run -dbname $DB:$magicDSAdvance_DB + + $magicDSAdvance_DB ++ + if ($magicDSAdvance_DB >= $DB:n) set magicDSAdvance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.revert.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 20 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/destreak.revert.log + + task.exec + $run = magicdstool -torevert -stage $DS_STAGE:$magicDSRevertStage + $magicDSRevertStage ++ + if ($magicDSRevertStage >= $DS_STAGE:n) set magicDSRevertStage = 0 + + if ($DB:n == 0) + option DEFAULT + else + + # save the DB name for the exit tasks + option $DB:$magicDSToRevert_DB + $run = $run -dbname $DB:$magicDSToRevert_DB + + # only bump database number after we have gone through all of the stages + if ($magicDSRevertStage == 0) + $magicDSToRevert_DB ++ + if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0 + end + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicDSToRevert + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicDSToRevert + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.revert.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages magicDSToRevert -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicDSToRevert $pageName pantaskState RUN + book getword magicDSToRevert $pageName exp_id -var EXP_ID + book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicDSToRevert $pageName camera -var CAMERA + book getword magicDSToRevert $pageName stage -var STAGE + book getword magicDSToRevert $pageName stage_id -var STAGE_ID + book getword magicDSToRevert $pageName component -var COMPONENT + book getword magicDSToRevert $pageName path_base -var PATH_BASE + book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE + book getword magicDSToRevert $pageName outroot -var OUTROOT + book getword magciDSToRevert $pageName bytes -var BYTES + book getword magciDSToRevert $pageName md5sum -var md5sum +# book getword magicDSToRevert $pageName recoveryroot -var RECROOT + book getword magicDSToRevert $pageName re_place -var REPLACE + book getword magicDSToRevert $pageName dbname -var DBNAME + + sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT + + substr $COMPONENT 0 3 COMP_HEAD + if ("$COMP_HEAD" == "sky") + set.host.for.skycell $COMPONENT + else + # assume component is a class_id, if not we will default to anyhost + set.host.for.camera $CAMERA $COMPONENT + end + + $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit 0 + process_exit magicDSToRevert $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicDSToRevert $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicDSToRevert $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicDSToRevert $options:0 pantaskState TIMEOUT + end +end + +task destreak.completed.revert + # task to finish processing for magicDSRuns being reverted or restored + host local + + periods -poll $LOADPOLL + #periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 20 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/destreak.completed.revert.log + + task.exec + $run = magicdstool -completedrevert + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicDSCompletedRevert_DB + + $run = $run -dbname $DB:$magicDSCompletedRevert_DB + + $magicDSCompletedRevert_DB ++ + if ($magicDSCompletedRevert_DB >= $DB:n) set magicDSCompletedRevert_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: /branches/eam_branches/20090715/ippTasks/diff.pro =================================================================== --- /branches/eam_branches/20090715/ippTasks/diff.pro (revision 25624) +++ /branches/eam_branches/20090715/ippTasks/diff.pro (revision 25625) @@ -14,4 +14,5 @@ ### Database lists $diffSkycell_DB = 0 +$diffAdvance_DB = 0 #$diffCleanup_DB = 0 @@ -36,4 +37,7 @@ active true end + task diff.advance + active true + end end @@ -44,4 +48,7 @@ end task diff.skycell.run + active false + end + task diff.advance active false end @@ -198,4 +205,56 @@ end end + + +# Advance exposures which have completed +task diff.advance + host local + + periods -poll $LOADPOLL +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/diff.advance.log + + task.exec + if ($LABEL:n == 0) break + $run = difftool -advance + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffAdvance_DB + $run = $run -dbname $DB:$diffAdvance_DB + $diffAdvance_DB ++ + if ($diffAdvance_DB >= $DB:n) set diffAdvance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + # # select images ready for diff analysis Index: /branches/eam_branches/20090715/ippTasks/dist.pro =================================================================== --- /branches/eam_branches/20090715/ippTasks/dist.pro (revision 25624) +++ /branches/eam_branches/20090715/ippTasks/dist.pro (revision 25625) @@ -16,4 +16,17 @@ $distQueue_DB = 0 +### list of stages +#list of stages +$DIST_STAGE:n = 0 +list DIST_STAGE -add "raw" +list DIST_STAGE -add "chip" +list DIST_STAGE -add "camera" +list DIST_STAGE -add "fake" +list DIST_STAGE -add "warp" +list DIST_STAGE -add "diff" +list DIST_STAGE -add "stack" + +$currentStage = 0 + ### Check status of tasks macro dist.status @@ -42,10 +55,6 @@ active true end - task dist.queueruns -# We aren't ready to run this task yet. It will queue much too much -# active true - active false - end -end +end + macro dist.off task dist.process.load @@ -59,7 +68,4 @@ end task dist.advance.run - active false - end - task dist.queueruns active false end @@ -74,9 +80,13 @@ npending 1 - stdout NULL - stderr $LOGDIR/dist.process.log task.exec - $run = disttool -pendingcomponent + # stdout NULL + # stderr $LOGSUBDIR/dist.process.load.log + + $run = disttool -pendingcomponent -stage $DIST_STAGE:$currentStage + $currentStage ++ + if ($currentStage >= $DIST_STAGE:n) set currentStage = 0 + if ($DB:n == 0) option DEFAULT @@ -85,6 +95,11 @@ option $DB:$distToProcess_DB $run = $run -dbname $DB:$distToProcess_DB - $distToProcess_DB ++ - if ($distToProcess_DB >= $DB:n) set distToProcess_DB = 0 + + # only increment the database number after we have gone through all of + # the stages + if ($currentStage == 0) + $distToProcess_DB ++ + if ($distToProcess_DB >= $DB:n) set distToProcess_DB = 0 + end end add_poll_args run @@ -134,9 +149,11 @@ if ("$pageName" == "NULL") break +# echo running $pageName + book setword distToProcess $pageName pantaskState RUN book getword distToProcess $pageName dist_id -var DIST_ID book getword distToProcess $pageName camera -var CAMERA - book getword distToProcess $pageName stage -var STAGE - book getword distToProcess $pageName stage_id -var STAGE_ID + book getword distToProcess $pageName stage -var DIST_STAGE + book getword distToProcess $pageName stage_id -var DIST_STAGE_ID book getword distToProcess $pageName clean -var CLEAN book getword distToProcess $pageName component -var COMPONENT @@ -159,6 +176,6 @@ $EXTRA_ARGS = $EXTRA_ARGS --magicked end - # no_magic is output as integer due to the union in the sql - if ($NO_MAGIC) + # is this right for stack and fake? + if ("$NO_MAGIC" == "T") $EXTRA_ARGS = $EXTRA_ARGS --no_magic end @@ -170,5 +187,4 @@ # set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR # host anyhost - substr $COMPONENT 0 3 COMP_HEAD @@ -180,10 +196,9 @@ end - sprintf logfile "%s/dist.%s.%s.log" $OUTDIR $DIST_ID $COMPONENT - stdout $logfile - stderr $logfile - - $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile +# stdout $logfile +# stderr $logfile + + $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $DIST_STAGE --stage_id $DIST_STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile add_standard_args run @@ -211,5 +226,4 @@ end - task dist.advance.load host local @@ -220,8 +234,7 @@ npending 1 - stdout NULL - stderr $LOGDIR/dist.advance.load.log - task.exec + # stderr $LOGSUBDIR/dist.advance.load.log + $run = disttool -toadvance if ($DB:n == 0) @@ -261,4 +274,5 @@ end end + task dist.advance.run @@ -291,6 +305,6 @@ sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID - stdout $logfile - stderr $logfile +# stdout $logfile +# stderr $logfile $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile @@ -318,64 +332,2 @@ end end - -task dist.queueruns -# host local - - periods -poll $RUNPOLL - periods -exec 30 - periods -timeout 45 - npending 1 - - active false - -# stdout $LOGDIR/dist.queuruns -# stderr $LOGDIR/dist.queueruns - - task.exec - $MYARGS = "" - # assume that we need magic unless we are running simtest - if ($?SIMTEST_CAMERA != 0) - $MYARGS = --no_magic - end - - $run = dist_queue_runs.pl $MYARGS --stage_limit 16 --logfile $LOGDIR/dist.queueruns - - if ($DB:n == 0) - $DBNAME = DEFAULT - else - # save the DB name for add_standard_args - $DBNAME = $DB:$distQueue_DB - $distQueue_DB ++ - if ($distQueue_DB >= $DB:n) set distQueue_DB = 0 - end - - host anyhost - - add_standard_args run - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - task.exit $EXIT_SUCCESS - # nothing to do - end - - # default exit status - task.exit default - showcommand failure - end - - # operation timed out? - task.exit timeout - showcommand timeout - end - - # operation timed out? - task.exit crash - showcommand crash - end -end Index: /branches/eam_branches/20090715/ippTasks/ipphosts.mhpcc.config =================================================================== --- /branches/eam_branches/20090715/ippTasks/ipphosts.mhpcc.config (revision 25624) +++ /branches/eam_branches/20090715/ippTasks/ipphosts.mhpcc.config (revision 25625) @@ -3,38 +3,51 @@ ipphosts METADATA camera STR skycell - count S32 12 - sky00 STR ipp024 - sky01 STR ipp023 - sky02 STR ipp026 - sky03 STR ipp028 - sky04 STR ipp029 - sky05 STR ipp030 - sky06 STR ipp031 - sky07 STR ipp032 - sky08 STR ipp033 - sky09 STR ipp034 - sky10 STR ipp035 - sky11 STR ipp036 -END - -ipphosts METADATA - camera STR MEGACAM - ccd00 STR po00 - ccd01 STR po01 - ccd02 STR po02 - ccd03 STR po03 - ccd04 STR po04 - ccd05 STR po05 - ccd06 STR po06 - ccd07 STR po07 - ccd08 STR po08 - ccd09 STR po09 -END - -ipphosts METADATA - camera STR CFH12K - ccd00 STR po10 - ccd01 STR po11 - ccd02 STR po12 + count S32 46 + sky00 STR ipp006 + sky01 STR ipp007 + sky02 STR ipp008 + sky03 STR ipp009 + sky04 STR ipp010 + sky05 STR ipp011 + sky06 STR ipp012 + sky07 STR ipp013 + sky08 STR ipp014 + sky09 STR ipp015 + sky10 STR ipp016 + sky11 STR ipp017 + sky12 STR ipp018 + sky13 STR ipp019 + sky14 STR ipp020 + sky15 STR ipp021 + sky16 STR ipp023 + sky17 STR ipp024 + sky18 STR ipp025 + sky19 STR ipp026 + sky20 STR ipp027 + sky21 STR ipp028 + sky22 STR ipp029 + sky23 STR ipp030 + sky24 STR ipp031 + sky25 STR ipp032 + sky26 STR ipp033 + sky27 STR ipp034 + sky28 STR ipp035 + sky29 STR ipp036 + sky30 STR ipp038 + sky31 STR ipp039 + sky32 STR ipp040 + sky33 STR ipp041 + sky34 STR ipp042 + sky35 STR ipp043 + sky36 STR ipp044 + sky37 STR ipp045 + sky38 STR ipp046 + sky39 STR ipp047 + sky40 STR ipp048 + sky41 STR ipp049 + sky42 STR ipp050 + sky43 STR ipp051 + sky44 STR ipp052 + sky45 STR ipp053 END Index: /branches/eam_branches/20090715/ippTasks/magic.pro =================================================================== --- /branches/eam_branches/20090715/ippTasks/magic.pro (revision 25624) +++ /branches/eam_branches/20090715/ippTasks/magic.pro (revision 25625) @@ -10,21 +10,8 @@ book init magicToTree book init magicToProcess -book init magicToDS -book init magicDSToRevert ### Database lists $magicToTree_DB = 0 $magicToProcess_DB = 0 -$magicToDS_DB = 0 -$magicDSToRevert_DB = 0 - -#list of stages -$STAGE:n = 0 -list STAGE -add "raw" -list STAGE -add "chip" -list STAGE -add "camera" -list STAGE -add "warp" -list STAGE -add "diff" -$magicDSRevertStage = 0 ### Check status of tasks @@ -32,6 +19,4 @@ book listbook magicToTree book listbook magicToProcess - book listbook magicToDS - book listbook magicDSToRevert end @@ -40,6 +25,4 @@ book init magicToTree book init magicToProcess - book init magicToDS - book init magicDSToRevert end @@ -58,20 +41,6 @@ active true end - task magic.destreak.load - active true - end - task magic.destreak.run - active true - end -end - -macro magic.ds.revert.on - task magic.ds.revert.load - active true - end - task magic.ds.revert.run - active true - end -end +end + ### Turn tasks off macro magic.off @@ -88,46 +57,5 @@ active false end - task magic.destreak.load - active false - end - task magic.destreak.run - active false - end -end - -macro magic.ds.revert.off - task magic.ds.revert.load - active false - end - task magic.ds.revert.run - active false - end -end -macro magic.ds.off - task magic.destreak.load - active false - end - task magic.destreak.run - active false - end -end -macro magic.ds.on - task magic.destreak.load - active true - end - task magic.destreak.run - active true - end -end - -macro magic.ds.status - echo magicToDS - book listbook magicToDS - echo "" - echo magicDSToRevert - book listbook magicDSToRevert -end - - +end task magic.tree.load @@ -400,279 +328,2 @@ end end - -task magic.destreak.load - host local - - periods -poll $LOADPOLL - # this query can take a long time - periods -exec 10 - periods -timeout 120 - npending 1 - - stdout NULL - stderr $LOGDIR/magic.destreak.log - - task.exec - $run = magicdstool -todestreak -limit 120 - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$magicToDS_DB - $run = $run -dbname $DB:$magicToDS_DB - $magicToDS_DB ++ - if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0 - end - add_poll_args run - add_poll_labels run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook magicToDS - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup magicToDS - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task magic.destreak.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - - task.exec - book npages magicToDS -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images (pantaskState == INIT) - book getpage magicToDS 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword magicToDS $pageName pantaskState RUN - book getword magicToDS $pageName exp_id -var EXP_ID - book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID - book getword magicToDS $pageName camera -var CAMERA - book getword magicToDS $pageName streaks_uri -var STREAKS - book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS - book getword magicToDS $pageName stage -var STAGE - book getword magicToDS $pageName stage_id -var STAGE_ID - book getword magicToDS $pageName component -var COMPONENT - book getword magicToDS $pageName uri -var URI - book getword magicToDS $pageName path_base -var PATH_BASE - book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE - book getword magicToDS $pageName outroot -var OUTROOT - book getword magicToDS $pageName recoveryroot -var RECROOT - book getword magicToDS $pageName re_place -var REPLACE - book getword magicToDS $pageName dbname -var DBNAME - - sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT - - substr $COMPONENT 0 3 COMP_HEAD - if ("$COMP_HEAD" == "sky") - set.host.for.skycell $COMPONENT - else - # assume component is a class_id, if not we will default to anyhost - set.host.for.camera $CAMERA $COMPONENT - end - - # TODO: do not add recoveryroot or replace if they are null or zero - - $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE - - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit 0 - process_exit magicToDS $options:0 $JOB_STATUS - end - - # locked list - task.exit default - showcommand failure - process_exit magicToDS $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword magicToDS $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword magicToDS $options:0 pantaskState TIMEOUT - end -end - -task magic.ds.revert.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 20 - npending 1 - active false - - stdout NULL - stderr $LOGDIR/magic.ds.revert.log - - task.exec - $run = magicdstool -torevert -stage $STAGE:$magicDSRevertStage - $magicDSRevertStage ++ - if ($magicDSRevertStage >= $STAGE:n) set magicDSRevertStage = 0 - - if ($DB:n == 0) - option DEFAULT - else - - # save the DB name for the exit tasks - option $DB:$magicDSToRevert_DB - $run = $run -dbname $DB:$magicDSToRevert_DB - - # only bump database number after we have gone through all of the stages - if ($magicDSRevertStage == 0) - $magicDSToRevert_DB ++ - end - if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0 - end - add_poll_args run - add_poll_labels run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook magicDSToRevert - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup magicDSToRevert - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task magic.ds.revert.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active false - - task.exec - book npages magicDSToRevert -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images (pantaskState == INIT) - book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword magicDSToRevert $pageName pantaskState RUN - book getword magicDSToRevert $pageName exp_id -var EXP_ID - book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID - book getword magicDSToRevert $pageName camera -var CAMERA - book getword magicDSToRevert $pageName stage -var STAGE - book getword magicDSToRevert $pageName stage_id -var STAGE_ID - book getword magicDSToRevert $pageName component -var COMPONENT - book getword magicDSToRevert $pageName path_base -var PATH_BASE - book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE - book getword magicDSToRevert $pageName outroot -var OUTROOT - book getword magciDSToRevert $pageName bytes -var BYTES - book getword magciDSToRevert $pageName md5sum -var md5sum -# book getword magicDSToRevert $pageName recoveryroot -var RECROOT - book getword magicDSToRevert $pageName re_place -var REPLACE - book getword magicDSToRevert $pageName dbname -var DBNAME - - sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT - - substr $COMPONENT 0 3 COMP_HEAD - if ("$COMP_HEAD" == "sky") - set.host.for.skycell $COMPONENT - else - # assume component is a class_id, if not we will default to anyhost - set.host.for.camera $CAMERA $COMPONENT - end - - $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE - - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit 0 - process_exit magicDSToRevert $options:0 $JOB_STATUS - end - - # locked list - task.exit default - showcommand failure - process_exit magicDSToRevert $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword magicDSToRevert $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword magicDSToRevert $options:0 pantaskState TIMEOUT - end -end - Index: /branches/eam_branches/20090715/ippTasks/pantasks.pro =================================================================== --- /branches/eam_branches/20090715/ippTasks/pantasks.pro (revision 25624) +++ /branches/eam_branches/20090715/ippTasks/pantasks.pro (revision 25625) @@ -32,5 +32,5 @@ # very basic values: set these with init.copy.mhpcc -$default_host = anyhost +$default_host = any $workdir_template = `pwd` Index: /branches/eam_branches/20090715/ippTasks/pstamp.pro =================================================================== --- /branches/eam_branches/20090715/ippTasks/pstamp.pro (revision 25624) +++ /branches/eam_branches/20090715/ippTasks/pstamp.pro (revision 25625) @@ -1,4 +1,8 @@ - -#$VERBOSE = 3 +# pstamp.pro : Postage Stamp Server tasks + +check.globals + +$LOGSUBDIR = $LOGDIR/pstamp +mkdir $LOGSUBDIR # these variables wil cycle through the known database names @@ -8,4 +12,5 @@ $pstampJob_DB = 0 $pstampFin_DB = 0 +$pstampRev_DB = 0 # set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config @@ -52,5 +57,9 @@ active true end -end + task pstamp.job.revert + active true + end +end + macro pstamp.off @@ -75,4 +84,18 @@ end task pstamp.job.run + active false + end + task pstamp.job.revert + active false + end +end + +macro pstamp.revert.on + task pstamp.job.revert + active true + end +end +macro pstamp.revert.off + task pstamp.job.revert active false end @@ -88,4 +111,6 @@ task.exec + stdout $LOGSUBDIR/pstamp.request.find.log + stderr $LOGSUBDIR/pstamp.request.find.log if ($DB:n == 0) option DEFAULT @@ -125,4 +150,6 @@ task.exec + stdout $LOGSUBDIR/pstamp.request.load.log + stderr $LOGSUBDIR/pstamp.request.load.log $run = pstamptool -pendingreq if ($DB:n == 0) @@ -168,4 +195,6 @@ task.exec + stdout $LOGSUBDIR/pstamp.request.run.log + stderr $LOGSUBDIR/pstamp.request.run.log book npages pstampRequest -var N if ($N == 0) break @@ -223,4 +252,6 @@ task.exec + stdout $LOGSUBDIR/pstamp.finish.load.log + stderr $LOGSUBDIR/pstamp.finish.load.log $run = pstamptool -completedreq if ($DB:n == 0) @@ -265,4 +296,6 @@ task.exec + stdout $LOGSUBDIR/request.finish.run.log + stderr $LOGSUBDIR/request.finish.run.log book npages pstampFinish -var N if ($N == 0) break @@ -320,4 +353,6 @@ task.exec + stdout $LOGSUBDIR/pstamp.job.load.log + stderr $LOGSUBDIR/pstamp.job.load.log $run = pstamptool -pendingjob if ($DB:n == 0) @@ -346,15 +381,16 @@ end + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end + task.exit default showcommand failure end - task.exit crash - showcommand crash - end - - task.exit timeout - showcommand timeout - end end @@ -365,4 +401,6 @@ task.exec + stdout $LOGSUBDIR/pstamp.job.run.log + stderr $LOGSUBDIR/pstamp.job.run.log book npages pstampJob -var N if ($N == 0) @@ -430,2 +468,44 @@ end end + +task pstamp.job.revert + host local + + periods -poll $LOADPOLL + periods -exec 10 + periods -timeout 300 + npending 1 + + task.exec + stdout $LOGSUBDIR/pstamp.job.revert.log + stderr $LOGSUBDIR/pstamp.job.revert.log + $run = pstamptool -revertjob -all + if ($DB:n == 0) + option DEFAULT + else + $run = $run $PS_DBSERVER -dbname $DB:$pstampRev_DB + $pstampRev_DB ++ + if ($pstampRev_DB >= $DB:n) set pstampRev_DB = 0 + end + add_poll_args run + # add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + # nothing to do + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + Index: /branches/eam_branches/20090715/ippTasks/rcserver.pro =================================================================== --- /branches/eam_branches/20090715/ippTasks/rcserver.pro (revision 25624) +++ /branches/eam_branches/20090715/ippTasks/rcserver.pro (revision 25625) @@ -4,5 +4,5 @@ check.globals -#$LOGSUBDIR = $LOGDIR/rcserver +$LOGSUBDIR = $LOGDIR/rcserver mkdir $LOGSUBDIR @@ -66,5 +66,5 @@ stdout NULL - stderr $LOGDIR/rcserver.makefileset.load.log + stderr $LOGSUBDIR/rcserver.makefileset.load.log task.exec @@ -86,5 +86,5 @@ task.exit 0 # convert 'stdout' to book format - ipptool2book stdout rcPendingFS -key dist_id:prod_id -uniq -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout rcPendingFS -key dist_id:dest_id -uniq -setword dbname $options:0 -setword pantaskState INIT if ($VERBOSE > 2) book listbook rcPendingFS @@ -131,5 +131,5 @@ book getword rcPendingFS $pageName dist_dir -var DIST_DIR # book getword rcPendingFS $pageName clean -var CLEAN - book getword rcPendingFS $pageName prod_id -var PROD_ID + book getword rcPendingFS $pageName dest_id -var DEST_ID book getword rcPendingFS $pageName product_name -var PRODUCT_NAME book getword rcPendingFS $pageName ds_dbhost -var DS_DBHOST @@ -141,5 +141,5 @@ host anyhost - sprintf logfile "%s/makefs.%s.%s.log" $DIST_DIR $DIST_ID $PROD_ID + sprintf logfile "%s/makefs.%s.%s.log" $DIST_DIR $DIST_ID $DEST_ID stdout $logfile stderr $logfile @@ -147,5 +147,5 @@ book setword rcPendingFS $pageName pantaskState RUN - $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --prod_id $PROD_ID --product_name $PRODUCT_NAME --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR + $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --dest_id $DEST_ID --product_name $PRODUCT_NAME --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR add_standard_args run @@ -184,5 +184,5 @@ stdout NULL - stderr $LOGDIR/rcserver.checkstatus.load.log + stderr $LOGSUBDIR/rcserver.checkstatus.load.log task.exec @@ -241,9 +241,9 @@ stdout NULL - stderr $LOGDIR/rcserver.checkstatus.run.log + stderr $LOGSUBDIR/rcserver.checkstatus.run.log book setword rcPendingDest $pageName pantaskState RUN book getword rcPendingDest $pageName dest_id -var DEST_ID - book getword rcPendingDest $pageName prod_id -var PROD_ID + book getword rcPendingDest $pageName dest_id -var DEST_ID book getword rcPendingDest $pageName status_uri -var STATUS_URI book getword rcPendingDest $pageName last_fileset -var LAST_FILESET @@ -252,5 +252,5 @@ host anyhost - $run = rcserver_checkstatus.pl --dest_id $DEST_ID --prod_id $PROD_ID --status_uri $STATUS_URI --last_fileset $LAST_FILESET + $run = rcserver_checkstatus.pl --dest_id $DEST_ID --dest_id $DEST_ID --status_uri $STATUS_URI --last_fileset $LAST_FILESET add_standard_args run Index: /branches/eam_branches/20090715/ippTasks/register.pro =================================================================== --- /branches/eam_branches/20090715/ippTasks/register.pro (revision 25624) +++ /branches/eam_branches/20090715/ippTasks/register.pro (revision 25625) @@ -217,4 +217,6 @@ book getword regPendingImfile $pageName tmp_class_id -var TMP_CLASS_ID book getword regPendingImfile $pageName uri -var URI + book getword regPendingImfile $pageName bytes -var BYTES + book getword regPendingImfile $pageName md5sum -var MD5SUM book getword regPendingImfile $pageName workdir -var WORKDIR_TEMPLATE book getword regPendingImfile $pageName dbname -var DBNAME @@ -244,5 +246,5 @@ # XXX register_imfile.pl differs from the standard script : it does not have an 'outroot' argument, and it does not take '--redirect' - $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI --logfile $logfile + $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI --logfile $logfile --bytes $BYTES --md5sum $MD5SUM add_standard_args run Index: /branches/eam_branches/20090715/ippTasks/site.mhpcc.pro =================================================================== --- /branches/eam_branches/20090715/ippTasks/site.mhpcc.pro (revision 25624) +++ /branches/eam_branches/20090715/ippTasks/site.mhpcc.pro (revision 25625) @@ -66,5 +66,5 @@ if ("$1" == "on") $NEBULOUS = 1 - $default_host = ipp023 + $default_host = any $workdir_template = neb://@HOST@.0 else