Index: trunk/ippTasks/Makefile.am =================================================================== --- trunk/ippTasks/Makefile.am (revision 42792) +++ trunk/ippTasks/Makefile.am (revision 42836) @@ -51,4 +51,7 @@ remote.pro \ fpcamera.pro \ + xcstack.pro \ + xcsky.pro \ + xccal.pro \ fullforce.pro Index: trunk/ippTasks/xccal.pro =================================================================== --- trunk/ippTasks/xccal.pro (revision 42836) +++ trunk/ippTasks/xccal.pro (revision 42836) @@ -0,0 +1,269 @@ +## xccal.pro : tasks for static sky calibration analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the +## xccal analysis. After a xccalRun entry (with +## associated xccal_id) is defined, psastro is performed (via script +## xccal.pl) (tasks in xccalRun). + +# test for required global variables +check.globals + +# skcal.pro should have a more restricted polling limit (to avoid stress with getstar) +# XXX: is this necessary anymore? +if ($?POLL_LIMIT_XCCAL == 0) set POLL_LIMIT_XCCAL = 64 + +macro set.xccal.poll + if ($0 != 2) + echo "USAGE:set.xccal.poll (value)" + break + end + + $POLL_LIMIT_XCCAL = $1 +end + +macro get.xccal.poll + echo "xccal poll limit: $POLL_LIMIT_XCCAL" +end + +### Initialise the books containing the tasks to do +book init xccalRun + +### Database lists +$xccal_DB = 0 +$xccal_revert_DB = 0 + +### Check status of xccal tasks +macro xccal.status + book listbook xccalRun +end + +### Reset xccal tasks +macro xccal.reset + book init xccalRun +end + +### Turn xccal tasks on +macro xccal.on + task xccal.load + active true + end + task xccal.run + active true + end +end + +### Turn xccal tasks off +macro xccal.off + task xccal.load + active false + end + task xccal.run + active false + end +end + +macro xccal.revert.on + task xccal.revert + active true + end +end + +macro xccal.revert.off + task xccal.revert + active false + end +end + +### Load tasks for xccal +### Tasks are loaded into xccalRun. +task xccal.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/xccal.log + + task.exec + if ($LABEL:n == 0) break + $run = xcskytool -pendingxccalrun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xccal_DB + $run = $run -dbname $DB:$xccal_DB + $xccal_DB ++ + if ($xccal_DB >= $DB:n) set xccal_DB = 0 + end + add_poll_args run + add_poll_labels run + $run = $run -limit $POLL_LIMIT_XCCAL + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout xccalRun -key xccal_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook xccalRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup xccalRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the xccal analysis +### Tasks are taken from xccalRun. +task xccal.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run use "long" exectime + periods -exec $RUNEXEC + book npages xccalRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + + # look for new entries in xccalRun (pantaskState == INIT) + book getpage xccalRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword xccalRun $pageName pantaskState RUN + book getword xccalRun $pageName xccal_id -var XCCAL_ID + book getword xccalRun $pageName xcstack_id -var XCSTACK_ID + book getword xccalRun $pageName tess_id -var TESS_DIR + book getword xccalRun $pageName skycell_id -var SKYCELL_ID + book getword xccalRun $pageName workdir -var WORKDIR_TEMPLATE + book getword xccalRun $pageName path_base -var PATH_BASE + book getword xccalRun $pageName reduction -var REDUCTION + book getword xccalRun $pageName state -var RUN_STATE + book getword xccalRun $pageName dbname -var DBNAME + + # move this above when we get camera into the db correctly: + # book getword xcskyResult $pageName camera -var CAMERA + $CAMERA = GPC1 + # $CAMERA = MEGACAM + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.stk.%s.xccal.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $XCSTACK_ID $XCCAL_ID + + stdout $LOGDIR/xccal.log + stderr $LOGDIR/xccal.log + + $run = xccal.pl --xccal_id $XCCAL_ID --outroot $outroot --redirect-output --camera $CAMERA --path_base $PATH_BASE --xcstack_id $XCSTACK_ID --run-state $RUN_STATE + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # since we have work to do shorten exec time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit xccalRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword xccalRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword xccalRun $options:0 pantaskState TIMEOUT + end +end + +task xccal.revert + host local + + periods -poll 10.0 + periods -exec 1200.0 + periods -timeout 120.0 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + $run = xcskytool -revertxccal -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xccal_revert_DB + $run = $run -dbname $DB:$xccal_revert_DB + $xccal_revert_DB ++ + if ($xccal_revert_DB >= $DB:n) set xccal_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: trunk/ippTasks/xcff.pro =================================================================== --- trunk/ippTasks/xcff.pro (revision 42836) +++ trunk/ippTasks/xcff.pro (revision 42836) @@ -0,0 +1,464 @@ +## xcff.pro : tasks for forced-photometry on the xcstack input warps : -*- sh -*- + +## This file contains panTasks definitions for performing the xcff analysis. + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init xcffRun +book init xcffSummary + +### Database lists +$xcff_DB = 0 +$xcff_revert_DB = 0 +$xcffSummary_DB = 0 +$xcffSummary_revert_DB = 0 + +### Check status of xcff tasks +macro xcff.status + book listbook xcffRun +end + +### Reset xcff tasks +macro xcff.reset + book init xcffRun +end +macro xcff.summary.status + book listbook xcffSummary +end + +### Reset xcff tasks +macro xcff.summary.reset + book init xcffSummary +end + +### Turn xcff tasks on +macro xcff.on + task xcff.load + active true + end + task xcff.run + active true + end + task xcff.summary.load + active true + end + task xcff.summary.run + active true + end +end + +### Turn xcff tasks off +macro xcff.off + task xcff.load + active false + end + task xcff.run + active false + end + task xcff.summary.load + active false + end + task xcff.summary.run + active false + end +end + +macro xcff.revert.on + task xcff.revert + active true + end + task xcff.summary.revert + active true + end +end + +macro xcff.revert.off + task xcff.revert + active false + end + task xcff.summary.revert + active false + end +end + +### Load tasks for xcff +### Tasks are loaded into xcffRun. +task xcff.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/xcff.log + + task.exec + if ($LABEL:n == 0) break + $run = xcfftool -todo + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xcff_DB + $run = $run -dbname $DB:$xcff_DB + $xcff_DB ++ + if ($xcff_DB >= $DB:n) set xcff_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout xcffRun -key xcff_id:warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook xcffRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup xcffRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the xcff analysis +### Tasks are taken from xcffRun. +task xcff.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run use "long" exectime + periods -exec $RUNEXEC + book npages xcffRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + + # look for new entries in xcffRun (pantaskState == INIT) + book getpage xcffRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword xcffRun $pageName pantaskState RUN + book getword xcffRun $pageName xcff_id -var XCFF_ID + book getword xcffRun $pageName warp_id -var WARP_ID + book getword xcffRun $pageName sources_path_base -var SOURCES_PATH_BASE + book getword xcffRun $pageName tess_id -var TESS_DIR + book getword xcffRun $pageName skycell_id -var SKYCELL_ID + book getword xcffRun $pageName workdir -var WORKDIR_TEMPLATE + book getword xcffRun $pageName warp_path_base -var WARP_PATH_BASE + book getword xcffRun $pageName reduction -var REDUCTION + book getword xcffRun $pageName dbname -var DBNAME + + # move this above when we get camera into the db correctly: + # book getword xcffRun $pageName camera -var CAMERA + $CAMERA = GPC1 + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.wrp.%s.xcff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $WARP_ID $XCFF_ID + + stdout $LOGDIR/xcff.log + stderr $LOGDIR/xcff.log + + $run = xcff_warp.pl --xcff_id $XCFF_ID --warp_id $WARP_ID --outroot $outroot --redirect-output --camera $CAMERA --warp_path_base $WARP_PATH_BASE --sources_path_base $SOURCES_PATH_BASE --skycell_id $SKYCELL_ID --threads @MAX_THREADS@ + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # since we have work to do shorten exec time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit xcffRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword xcffRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword xcffRun $options:0 pantaskState TIMEOUT + end +end + +task xcff.revert + host local + + periods -poll 10.0 + periods -exec 1200.0 + periods -timeout 120.0 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/xcff.revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + $run = xcfftool -revert -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xcff_revert_DB + $run = $run -dbname $DB:$xcff_revert_DB + $xcff_revert_DB ++ + if ($xcff_revert_DB >= $DB:n) set xcff_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Load tasks for xcffSummary +### Tasks are loaded into xcffSummary. +task xcff.summary.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/xcff.summary.log + + task.exec + if ($LABEL:n == 0) break + $run = xcfftool -toadvance + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xcffSummary_DB + $run = $run -dbname $DB:$xcffSummary_DB + $xcffSummary_DB ++ + if ($xcffSummary_DB >= $DB:n) set xcffSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout xcffSummary -key xcff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook xcffSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup xcffSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the xcffSummary analyasis +### Tasks are taken from xcffSummary. +task xcff.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run use "long" exectime + periods -exec $RUNEXEC + book npages xcffSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + + # look for new entries in xcffSummary (pantaskState == INIT) + book getpage xcffSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword xcffSummary $pageName pantaskState RUN + book getword xcffSummary $pageName xcff_id -var XCFF_ID + book getword xcffSummary $pageName tess_id -var TESS_DIR + book getword xcffSummary $pageName skycell_id -var SKYCELL_ID + book getword xcffSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword xcffSummary $pageName reduction -var REDUCTION + book getword xcffSummary $pageName dbname -var DBNAME + + # move this above when we get camera into the db correctly: + # book getword xcffSummary $pageName camera -var CAMERA + $CAMERA = GPC1 + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.xcff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $XCFF_ID + + stdout $LOGDIR/xcff.summary.log + stderr $LOGDIR/xcff.summary.log + + $run = xcff_summary.pl --xcff_id $XCFF_ID --outroot $outroot --redirect-output --camera $CAMERA --threads @MAX_THREADS@ + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # since we have work to do shorten exec time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit xcffSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword xcffSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword xcffSummary $options:0 pantaskState TIMEOUT + end +end + +task xcff.summary.revert + host local + + periods -poll 10.0 + periods -exec 1200.0 + periods -timeout 120.0 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/xcff.summary.revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + $run = xcfftool -revertsummary -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xcffSummary_revert_DB + $run = $run -dbname $DB:$xcffSummary_revert_DB + $xcffSummary_revert_DB ++ + if ($xcffSummary_revert_DB >= $DB:n) set xcffSummary_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: trunk/ippTasks/xcsky.pro =================================================================== --- trunk/ippTasks/xcsky.pro (revision 42836) +++ trunk/ippTasks/xcsky.pro (revision 42836) @@ -0,0 +1,277 @@ +## xcsky.pro : tasks for xcsky photometry analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the +## xcsky photometry analysis. After a xcsky entry (with +## associated sky_id) is defined, psphotStack is performed (via script +## xcsky.pl) (tasks in xcskyResult). + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init xcskyResult + +### Database lists +$xcsky_DB = 0 +$xcsky_revert_DB = 0 + +### Check status of xcsky tasks +macro xcsky.status + book listbook xcskyResult +end + +### Reset xcsky tasks +macro xcsky.reset + book init xcskyResult +end + +### Turn xcsky tasks on +macro xcsky.on + task xcsky.load + active true + end + task xcsky.run + active true + end + task xcsky.revert + active false + end +end + +### Turn xcsky tasks off +macro xcsky.off + task xcsky.load + active false + end + task xcsky.run + active false + end + task xcsky.revert + active false + end +end + +macro xcsky.revert.on + task xcsky.revert + active true + end +end + +macro xcsky.revert.off + task xcsky.revert + active false + end +end + +$RA_POLL_MAX = 0 + +macro set.ra.max + if ($0 != 2) + echo "USAGE: set.ra.max (ra_max_deg)" + break + end + $RA_POLL_MAX = $1 +end +macro get.ra.max + echo $RA_POLL_MAX +end + +### Load tasks for xcsky +### Tasks are loaded into xcskyResult. +task xcsky.load + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/xcsky.log + + task.exec + if ($LABEL:n == 0) break + $run = xcskytool -todo + if ($RA_POLL_MAX > 0) + $run = $run -ra_max $RA_POLL_MAX + end + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xcsky_DB + $run = $run -dbname $DB:$xcsky_DB -limit 40 + $xcsky_DB ++ + if ($xcsky_DB >= $DB:n) set xcsky_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout xcskyResult -key xcsky_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook xcskyResult + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup xcskyResult + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the xcsky analysis +### Tasks are taken from xcskyResult. +task xcsky.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages xcskyResult -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new entries in xcskyResult (pantaskState == INIT) + book getpage xcskyResult 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword xcskyResult $pageName pantaskState RUN + book getword xcskyResult $pageName xcsky_id -var XCSKY_ID + book getword xcskyResult $pageName tess_id -var TESS_DIR + book getword xcskyResult $pageName skycell_id -var SKYCELL_ID + book getword xcskyResult $pageName workdir -var WORKDIR_TEMPLATE + book getword xcskyResult $pageName path_base -var PATH_BASE + book getword xcskyResult $pageName reduction -var REDUCTION + book getword xcskyResult $pageName dbname -var DBNAME + book getword xcskyResult $pageName state -var RUN_STATE + + # move this above when we get camera into the db correctly: + # book getword xcskyResult $pageName camera -var CAMERA + $CAMERA = GPC1 + # $CAMERA = MEGACAM + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + + if ("$PATH_BASE" == "NULL") + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.sky.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $XCSKY_ID + else + # run-state must be update use existing path_base + $outroot = $PATH_BASE + end + + stdout $LOGDIR/xcsky.log + stderr $LOGDIR/xcsky.log + + $run = xcsky.pl --xcsky_id $XCSKY_ID --run-state $RUN_STATE --outroot $outroot --redirect-output --camera $CAMERA --threads @MAX_THREADS@ + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are able to run the 'exec', use a short retry time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit xcskyResult $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword xcskyResult $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword xcskyResult $options:0 pantaskState TIMEOUT + end +end + +task xcsky.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it doesn't + # continue to occur). + $run = xcskytool -revert -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xcsky_revert_DB + $run = $run -dbname $DB:$xcsky_revert_DB + $xcsky_revert_DB ++ + if ($xcsky_revert_DB >= $DB:n) set xcsky_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: trunk/ippTasks/xcstack.pro =================================================================== --- trunk/ippTasks/xcstack.pro (revision 42836) +++ trunk/ippTasks/xcstack.pro (revision 42836) @@ -0,0 +1,277 @@ +## xcstack.pro : tasks for image xcstacking : -*- sh -*- + +## This file contains panTasks definitions for performing the image stacking. +## After a xcstack (with associated xcstack_id) is defined, the stack is performed +## (tasks in xcstackSumSkyfile). + +# test for required global variables +check.globals + +if ($?POLL_LIMIT_XCSTACK == 0) set POLL_LIMIT_XCSTACK = 100 + +macro set.xcstack.poll + if ($0 != 2) + echo "USAGE:set.xcstack.poll (value)" + break + end + + $POLL_LIMIT_XCSTACK = $1 +end +macro get.xcstack.poll + echo $POLL_LIMIT_XCSTACK +end + +### Initialise the books containing the tasks to do +book init xcstackSumSkyfile + +### Database lists +if (not($?xcstackSkycell_DB)) set xcstackSkycell_DB = 0 +if (not($?xcstack_revert_DB)) set xcstack_revert_DB = 0 + +### Check status of xcstacking tasks +macro xcstack.status + book listbook xcstackSumSkyfile +end + +### Reset xcstacking tasks +macro xcstack.reset + book init xcstackSumSkyfile +end + +### Turn xcstacking tasks on +macro xcstack.on + task xcstack.skycell.load + active true + end + task xcstack.skycell.run + active true + end + task xcstack.revert + active false + end +end + +### Turn xcstacking tasks off +macro xcstack.off + task xcstack.skycell.load + active false + end + task xcstack.skycell.run + active false + end + task xcstack.revert + active false + end +end + +macro xcstack.revert.on + task xcstack.revert + active true + end +end + +macro xcstack.revert.off + task xcstack.revert + active false + end +end + +### Load tasks for doing the xcstack +### Tasks are loaded into xcstackSumSkyfile. +task xcstack.skycell.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/xcstack.skycell.log + + task.exec + if ($LABEL:n == 0) break + $run = xcstacktool -tosum + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xcstackSkycell_DB + $run = $run -dbname $DB:$xcstackSkycell_DB + $xcstackSkycell_DB ++ + if ($xcstackSkycell_DB >= $DB:n) set xcstackSkycell_DB = 0 + end + add_poll_args run + add_poll_labels run + # change the limit (the last one on the command line takes precedence) + $run = $run -limit $POLL_LIMIT_XCSTACK + + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout xcstackSumSkyfile -key xcstack_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook xcstackSumSkyfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup xcstackSumSkyfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for xcstack jobs +### Tasks are taken from xcstackSumSkyfile. +task xcstack.skycell.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $LOADEXEC + + book npages xcstackSumSkyfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new images in xcstackSumSkyfile (pantaskState == INIT) + book getpage xcstackSumSkyfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword xcstackSumSkyfile $pageName pantaskState RUN + book getword xcstackSumSkyfile $pageName xcstack_id -var XCSTACK_ID + book getword xcstackSumSkyfile $pageName tess_id -var TESS_DIR + book getword xcstackSumSkyfile $pageName skycell_id -var SKYCELL_ID + book getword xcstackSumSkyfile $pageName workdir -var WORKDIR_TEMPLATE + book getword xcstackSumSkyfile $pageName path_base -var PATH_BASE + book getword xcstackSumSkyfile $pageName reduction -var REDUCTION + book getword xcstackSumSkyfile $pageName dbname -var DBNAME + book getword xcstackSumSkyfile $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # XXX old code: + # host anyhost + # $WORKDIR = $WORKDIR_TEMPLATE + + basename $TESS_DIR -var TESS_ID + if ("$PATH_BASE" == "NULL") + sprintf outroot "%s/%s/%s/%s.%s.xcstk.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $XCSTACK_ID + else + $outroot = $PATH_BASE + end + + stdout $LOGDIR/xcstack.skycell.log + stderr $LOGDIR/xcstack.skycell.log + + # XXX unclear how to set the camera here. needs to match an entry in ippconfig/system.conf + $run = xcstack_skycell.pl --camera GPC1 --threads @MAX_THREADS@ --xcstack_id $XCSTACK_ID --outroot $outroot --redirect-output --run-state $RUN_STATE + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + + # re-try quickly until we run out + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit xcstackSumSkyfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword xcstackSumSkyfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword xcstackSumSkyfile $options:0 pantaskState TIMEOUT + end +end + +task xcstack.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it doesn't + # continue to occur). + $run = xcstacktool -revertsumskyfile -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$xcstack_revert_DB + $run = $run -dbname $DB:$xcstack_revert_DB + $xcstack_revert_DB ++ + if ($xcstack_revert_DB >= $DB:n) set xcstack_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +