Index: /branches/eam_branch_20080109/ippTasks/.cvsignore =================================================================== --- /branches/eam_branch_20080109/ippTasks/.cvsignore (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/.cvsignore (revision 16023) @@ -0,0 +1,9 @@ +Makefile +Makefile.in +aclocal.m4 +autom4te.cache +config.log +config.status +configure +install-sh +missing Index: /branches/eam_branch_20080109/ippTasks/Makefile.am =================================================================== --- /branches/eam_branch_20080109/ippTasks/Makefile.am (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/Makefile.am (revision 16023) @@ -0,0 +1,33 @@ +task_files = \ + detrend.mkruns.pro \ + detrend.norm.pro \ + detrend.process.pro \ + detrend.reject.pro \ + detrend.resid.pro \ + detrend.stack.pro \ + mkdetrend.pro \ + pantasks.pro \ + register.pro \ + chip.pro \ + camera.pro \ + warp.pro \ + magic.pro \ + diff.pro \ + stack.pro \ + summit.copy.pro \ + testscript.pro + +detrend_files = \ + simtest.det \ + isp.det + +pantasksdir = $(datadir)/pantasks +ipptasksdir = $(pantasksdir)/modules +ipptasks_DATA = $(task_files) $(detrend_files) + +install-data-hook: + chmod 0755 $(panstasks) $(ipptasksdir) + +EXTRA_DIST = $(task_files) $(detrend_files) + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/eam_branch_20080109/ippTasks/autogen.sh =================================================================== --- /branches/eam_branch_20080109/ippTasks/autogen.sh (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/autogen.sh (revision 16023) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ippTasks +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +#($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $LIBTOOlIZE installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" +# DIE=1 +#} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +#($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $AUTOHEADER installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" +# DIE=1 +#} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +#$LIBTOOLIZE --copy --force || echo "$LIBTOOlIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +#$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/eam_branch_20080109/ippTasks/calibration.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/calibration.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/calibration.pro (revision 16023) @@ -0,0 +1,181 @@ +## this file contains the tasks for running the calibration stage +## these tasks use the books XXX + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/calibration +exec mkdir -p $LOGSUBDIR + +book init calBook + +macro calibration.reset + book init calBook +end + +macro calibration.status + book listbook calBook +end + +macro calibration.on + task calibration.init + active true + end +end + +macro calibration.off + task calibration.init + active false + end +end + +# these variables will cycle through the known database names +$calInit_DB = 0 + +# create new calibration entries for the currently known DVO databases +# run this multiple times once an hour - on pass for each db +task calibration.init + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 60 + trange Hourly@00:00 Hourly@00:10 + # every hour on the hour + + if ($DB:n == 0) + npending $DB:n + else + npending 1 + end + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/calibration.log + + ## XXX have caltool define the regions, or have the task.exit below do it? + + # generate queues for all of the known (and active) DVO databases + task.exec + # get the current ST: + $RAs = 15*($ST - 1) + $RAe = 15*($ST - 2) + $DECs = -90.0 + $DECe = +90.0 + + $REGION = "$RAs,$RAe:$DECs,$DECe" + + if ($DB:n == 0) + option DEFAULT + command caltool -databases -active true + else + # save the DB name for the exit tasks + option $DB:$calInit_DB + option $REGION + command caltool -databases -dbname $DB:$calInit_DB -region $REGION + $calInit_DB ++ + if ($calInit_DB >= $DB:n) set calInit_DB = 0 + end + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout calBook -key dbname:dvo_id -uniq -setword dbname $options:0 -setword region $options:1 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook calBook + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup calBook + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create new calibration entries for the currently known DVO databases +# run this multiple times once an hour - on pass for each db +task calibration.resort + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 60 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/calibration.log + + task.exec + book npages calBook -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in calBook + # the sequencing in this task set is by the pantasksState (see notes.txt) + book getpage calBook 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword calBook $pageName pantaskState RUN + + # XXX probably need to set the output / log based on WORKDIR... + book getword calBook $pageName catdir -var CATDIR + book getword calBook $pageName region -var REGION + book getword calBook $pageName dbname -var DBNAME + + # specify choice of remote host + if ($parallel) + host anyhost + else + host local + end + + ## XXX generate output log based on filerule? + ## XXX put this in a function? + + # save the pageName for future reference below + options $pageName + + # XXX do we modify relphot to loop over the filters? + $run = calibrate_dvo.pl --catdir $CATDIR --region $REGION + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit calBook $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword calBook $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080109/ippTasks/camera.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/camera.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/camera.pro (revision 16023) @@ -0,0 +1,187 @@ +## this file contains the tasks for running the phase 3 stage +## these tasks use the book camPendingExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/camera +exec mkdir -p $LOGSUBDIR + +book init camPendingExp + +macro camera.status + book listbook camPendingExp +end + +macro camera.reset + book init camPendingExp +end + +macro camera.on + task camera.exp.load + active true + end + task camera.exp.run + active true + end +end + +macro camera.off + task camera.exp.load + active false + end + task camera.exp.run + active false + end +end + +# this variable will cycle through the known database names +$camera_DB = 0 + +# select images ready for camera analysis +# new entries are added to camPendingExp +# skip already-present entries +task camera.exp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout $LOGSUBDIR/camera.exp.log + stderr $LOGSUBDIR/camera.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command camtool -pendingexp -limit 20 + # echo command camtool -pendingexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$camera_DB + command camtool -pendingexp -limit 20 -dbname $DB:$camera_DB + # echo command camtool -pendingexp -limit 20 -dbname $DB:$camera_DB + $camera_DB ++ + if ($camera_DB >= $DB:n) set camera_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout camPendingExp -key cam_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook camPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup camPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the cameraexposure script on pending images +task camera.exp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 10 + + task.exec + book npages camPendingExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in camPendingExp (pantaskState == INIT) + book getpage camPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword camPendingExp $pageName pantaskState RUN + book getword camPendingExp $pageName camera -var CAMERA + book getword camPendingExp $pageName exp_tag -var EXP_TAG + book getword camPendingExp $pageName cam_id -var CAM_ID + book getword camPendingExp $pageName workdir -var WORKDIR + book getword camPendingExp $pageName dvodb -var DVODB + book getword camPendingExp $pageName dbname -var DBNAME + book getword camPendingExp $pageName reduction -var REDUCTION + + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end + + # output log from filerule + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s" $outpath $EXP_TAG + sprintf outroot "%s/%s.cm.%s" $outpath $EXP_TAG $CAM_ID + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot` + if ("$logfile" == "") + $logfile = $outroot.log + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = camera_exp.pl --exp_tag $EXP_TAG --cam_id $CAM_ID --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$DVODB" != "NULL") + $run = $run --dvodb $DVODB + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit camPendingExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword camPendingExp $options:0 pantaskState TIMEOUT + end +end + +## XXX add a global path to output files Index: /branches/eam_branch_20080109/ippTasks/chip.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/chip.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/chip.pro (revision 16023) @@ -0,0 +1,183 @@ +## this file contains the tasks for running the phase 0 stage +## these tasks use the book chipPendingImfile + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/chip +exec mkdir -p $LOGSUBDIR + +book init chipPendingImfile + +macro chip.status + book listbook chipPendingImfile +end + +macro chip.reset + book init chipPendingImfile +end + +macro chip.on + task chip.imfile.load + active true + end + task chip.imfile.run + active true + end +end + +macro chip.off + task chip.imfile.load + active false + end + task chip.imfile.run + active false + end +end + +# this variable will cycle through the known database names +$chip_DB = 0 + +# select images ready for chip analysis +# new entries are added to chipPendingImfile +# skip already-present entries +task chip.imfile.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/chip.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command chiptool -pendingimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$chip_DB + command chiptool -pendingimfile -limit 20 -dbname $DB:$chip_DB + $chip_DB ++ + if ($chip_DB >= $DB:n) set chip_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipPendingImfile -key chip_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipPendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup chipPendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the chip_imfile.pl script on pending images +task chip.imfile.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 60 + + task.exec + book npages chipPendingImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in chipPendingImfile (pantaskState == INIT) + book getpage chipPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipPendingImfile $pageName pantaskState RUN + book getword chipPendingImfile $pageName camera -var CAMERA + book getword chipPendingImfile $pageName exp_id -var EXP_ID + book getword chipPendingImfile $pageName exp_tag -var EXP_TAG + book getword chipPendingImfile $pageName chip_id -var CHIP_ID + book getword chipPendingImfile $pageName workdir -var WORKDIR + book getword chipPendingImfile $pageName class_id -var CLASS_ID + book getword chipPendingImfile $pageName uri -var URI + book getword chipPendingImfile $pageName dbname -var DBNAME + book getword chipPendingImfile $pageName reduction -var REDUCTION + + # specify choice of remote host + # XXX need to choose based on chips + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s" $outpath $EXP_TAG + sprintf outroot "%s/%s.ch.%s" $outpath $EXP_TAG $CHIP_ID + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` + if ("$logfile" == "") + sprintf logfile "%s.%s.log" $outroot $CLASS_ID + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = chip_imfile.pl --exp_id $EXP_ID --chip_id $CHIP_ID --class_id $CLASS_ID --exp_tag $EXP_TAG --uri $URI --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit chipPendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipPendingImfile $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080109/ippTasks/chiphosts.config =================================================================== --- /branches/eam_branch_20080109/ippTasks/chiphosts.config (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/chiphosts.config (revision 16023) @@ -0,0 +1,23 @@ + +chiphosts MULTI + +chiphosts METADATA + camera STR megacam + ccd00 STR po00 + ccd01 STR po01 + ccd02 STR po02 + ccd03 STR po03 + ccd04 STR po04 + ccd05 STR po05 + ccd06 STR po06 + ccd07 STR po07 + ccd08 STR po08 + ccd09 STR po09 +END + +chiphosts METADATA + camera STR cfh12k + ccd00 STR po10 + ccd01 STR po11 + ccd02 STR po12 +END Index: /branches/eam_branch_20080109/ippTasks/configure.ac =================================================================== --- /branches/eam_branch_20080109/ippTasks/configure.ac (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/configure.ac (revision 16023) @@ -0,0 +1,14 @@ +AC_PREREQ(2.59) + +AC_INIT([ippTasks], [1.1.0], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([pantasks.pro]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_MAINTAINER_MODE + +AC_PROG_INSTALL + +AC_CONFIG_FILES([ + Makefile +]) +AC_OUTPUT Index: /branches/eam_branch_20080109/ippTasks/detrend.correct.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/detrend.correct.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/detrend.correct.pro (revision 16023) @@ -0,0 +1,331 @@ +## this file contains the tasks for running the detrend correction stage +## these tasks use the books detPendingCorrectedImfile and detPendingCorrectedExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detPendingCorrectedImfile +book init detPendingCorrectedExp + +macro detcorr.reset + book init detPendingCorrectedImfile + book init detPendingCorrectedExp +end + +macro detcorr.status + echo detPendingCorrectedImfile + book listbook detPendingCorrectedImfile + echo detPendingCorrectedExp + book listbook detPendingCorrectedExp +end + +macro detcorr.on + task detrend.correct.load + active true + end + task detrend.correct.run + active true + end + task detrend.correctexp.load + active true + end + task detrend.correctexp.run + active true + end +end + +macro detcorr.off + task detrend.correct.load + active false + end + task detrend.correct.run + active false + end + task detrend.correctexp.load + active false + end + task detrend.correctexp.run + active false + end +end + + +# these variables will cycle through the known database names +$detPendingCorrectedImfile_DB = 0 +$detPendingCorrectedExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingCorrectedImfile +# compare the new list with the ones already selected +task detrend.correct.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detcorr.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tocorrectedimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingCorrectedImfile_DB + command dettool -tocorrectedimfile -limit 20 -dbname $DB:$detPendingCorrectedImfile_DB + $detPendingCorrectedImfile_DB ++ + if ($detPendingCorrectedImfile_DB >= $DB:n) set detPendingCorrectedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingCorrectedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingCorrectedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingCorrectedImfile + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand + end +end + +# run detrend_correct_imfile.pl on pending images +task detrend.correct.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingCorrectedImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingCorrectedImfile + book getpage detPendingCorrectedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingCorrectedImfile $pageName pantaskState RUN + book getword detPendingCorrectedImfile $pageName det_id -var DET_ID + book getword detPendingCorrectedImfile $pageName exp_id -var EXP_ID + book getword detPendingCorrectedImfile $pageName class_id -var CLASS_ID + book getword detPendingCorrectedImfile $pageName exp_tag -var EXP_TAG XXX where does this come from + book getword detPendingCorrectedImfile $pageName det_type -var DET_TYPE XXX where does this come from + book getword detPendingCorrectedImfile $pageName camera -var CAMERA XXX where does this come from + book getword detPendingCorrectedImfile $pageName uri -var URI + book getword detPendingCorrectedImfile $pageName corr_id -var CORR_ID + book getword detPendingCorrectedImfile $pageName corr_type -var CORR_TYPE + book getword detPendingCorrectedImfile $pageName workdir -var WORKDIR + book getword detPendingCorrectedImfile $pageName dbname -var DBNAME + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + ## output log filename + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detcorr.%s" $outpath $EXP_TAG $DET_ID + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` + if ("$logfile" == "") + sprintf logfile "%s.%s.log" $outroot $CLASS_ID + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_correct_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --corr_id $CORR_ID --corr_type $CORR_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingCorrectedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingCorrectedImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingCorrectedExp +# compare the new list with the ones already selected +task detrend.correctexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detcorr.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tocorrectedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingCorrectedExp_DB + command dettool -tocorrectedexp -limit 20 -dbname $DB:$detPendingCorrectedExp_DB + $detPendingCorrectedExp_DB ++ + if ($detPendingCorrectedExp_DB >= $DB:n) set detPendingCorrectedExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingCorrectedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingCorrectedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingCorrectedExp + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.correctexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingCorrectedExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new exposures in detPendingCorrectedExp + book getpage detPendingCorrectedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingCorrectedExp $pageName pantaskState RUN + book getword detPendingCorrectedExp $pageName det_id -var DET_ID + book getword detPendingCorrectedExp $pageName exp_id -var EXP_ID + book getword detPendingCorrectedExp $pageName det_type -var DET_TYPE + book getword detPendingCorrectedExp $pageName camera -var CAMERA + book getword detPendingCorrectedExp $pageName exp_tag -var EXP_TAG + book getword detPendingCorrectedExp $pageName corr_id -var CORR_ID + book getword detPendingCorrectedExp $pageName corr_type -var CORR_TYPE + book getword detPendingCorrectedExp $pageName workdir -var WORKDIR + book getword detPendingCorrectedExp $pageName dbname -var DBNAME + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # XXX push this into the filerules? + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detcorr.%s" $outpath $EXP_TAG $DET_ID + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot` + if ("$logfile" == "") + $logfile = $outroot.log + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_correct_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --corr_id $CORR_ID --corr_type $CORR_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit detPendingCorrectedExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingCorrectedExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080109/ippTasks/detrend.mkruns.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/detrend.mkruns.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/detrend.mkruns.pro (revision 16023) @@ -0,0 +1,97 @@ +## XXX this script needs to be updated to use books, not queues +## XXX this script needs to be reconsidered... + +## This script defines tasks which regularly initiate the detrend +## runs. Each detrend type is given its own task, which are run once +## (or twice) per day to build the relevant detrend data + +## when one of these tasks is run, it creates a new detrend run ID in +## the database, and selects the exposures for that run on the basis +## of the specified filters + +# create a new detrend bias-creation run +task mkdetruns.megacam + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.megacam + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new MEGACAM detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end + +# create a new detrend bias-creation run +task mkdetruns.isp + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.isp + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new ISP detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end + +# create a new detrend bias-creation run +task mkdetruns.gpc + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.gpc + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new GPC detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end Index: /branches/eam_branch_20080109/ippTasks/detrend.norm.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/detrend.norm.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/detrend.norm.pro (revision 16023) @@ -0,0 +1,445 @@ +## this file contains the tasks for running the detrend normalization stages +## these tasks use the books detPendingNormStatImfile detPendingNormImfile detPendingNormExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detPendingNormStatImfile +book init detPendingNormImfile +book init detPendingNormExp + +macro detnorm.reset + book init detPendingNormStatImfile + book init detPendingNormImfile + book init detPendingNormExp +end + +macro detnorm.status + book listbook detPendingNormStatImfile + book listbook detPendingNormImfile + book listbook detPendingNormExp +end + +macro detnorm.on + task detrend.norm.load + active true + end + task detrend.norm.run + active true + end + task detrend.normexp.load + active true + end + task detrend.normexp.run + active true + end + task detrend.normstat.load + active true + end + task detrend.normstat.run + active true + end +end + +macro detnorm.off + task detrend.norm.load + active false + end + task detrend.norm.run + active false + end + task detrend.normexp.load + active false + end + task detrend.normexp.run + active false + end + task detrend.normstat.load + active false + end + task detrend.normstat.run + active false + end +end + +# these variables will cycle through the known database names +$detPendingNormStatImfile_DB = 0 +$detPendingNormImfile_DB = 0 +$detPendingNormExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingNormStatImfile +# compare the new list with the ones already selected +task detrend.normstat.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detrend.normstat.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tonormalizedstat -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingNormStatImfile_DB + command dettool -tonormalizedstat -limit 20 -dbname $DB:$detPendingNormStatImfile_DB + $detPendingNormStatImfile_DB ++ + if ($detPendingNormStatImfile_DB >= $DB:n) set detPendingNormStatImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormStatImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormStatImfile + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.normstat.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingNormStatImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingNormStatImfile + book getpage detPendingNormStatImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormStatImfile $pageName pantaskState RUN + book getword detPendingNormStatImfile $pageName det_id -var DET_ID + book getword detPendingNormStatImfile $pageName det_type -var DET_TYPE + book getword detPendingNormStatImfile $pageName iteration -var ITERATION + book getword detPendingNormStatImfile $pageName camera -var CAMERA + book getword detPendingNormStatImfile $pageName workdir -var WORKDIR + book getword detPendingNormStatImfile $pageName dbname -var DBNAME + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + $outpath = $outpath/$CAMERA.$DET_TYPE.$DET_ID + $logfile = $outpath/$CAMERA.$DET_TYPE.norm.$DET_ID.$ITERATION.log + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormStatImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormStatImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingNormImfile +# compare the new list with the ones already selected +task detrend.norm.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detrend.norm.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tonormalize -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingNormImfile_DB + command dettool -tonormalize -limit 20 -dbname $DB:$detPendingNormImfile_DB + $detPendingNormImfile_DB ++ + if ($detPendingNormImfile_DB >= $DB:n) set detPendingNormImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormImfile + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.norm.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingNormImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingNormImfile + book getpage detPendingNormImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormImfile $pageName pantaskState RUN + book getword detPendingNormImfile $pageName det_type -var DET_TYPE + book getword detPendingNormImfile $pageName camera -var CAMERA + book getword detPendingNormImfile $pageName uri -var URI + book getword detPendingNormImfile $pageName det_id -var DET_ID + book getword detPendingNormImfile $pageName iteration -var ITERATION + book getword detPendingNormImfile $pageName class_id -var CLASS_ID + book getword detPendingNormImfile $pageName norm -var NORM + book getword detPendingNormImfile $pageName workdir -var WORKDIR + book getword detPendingNormImfile $pageName dbname -var DBNAME + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # XXX use ipp_filename.pl to lookup output file names + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + $outpath = $outpath/$CAMERA.$DET_TYPE.$DET_ID + $logfile = $outpath/$CAMERA.$DET_TYPE.norm.$DET_ID.$ITERATION.$CLASS_ID.log + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingNormExp +# compare the new list with the ones already selected +task detrend.normexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detrend.normexp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tonormalizedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingNormExp_DB + command dettool -tonormalizedexp -limit 20 -dbname $DB:$detPendingNormExp_DB + $detPendingNormExp_DB ++ + if ($detPendingNormExp_DB >= $DB:n) set detPendingNormExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.normexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingNormExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingNormExp + book getpage detPendingNormExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormExp $pageName pantaskState RUN + book getword detPendingNormExp $pageName det_id -var DET_ID + book getword detPendingNormExp $pageName iteration -var ITERATION + book getword detPendingNormExp $pageName det_type -var DET_TYPE + book getword detPendingNormExp $pageName camera -var CAMERA + book getword detPendingNormExp $pageName workdir -var WORKDIR + book getword detPendingNormExp $pageName dbname -var DBNAME + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # XXX add $WORKDIR/$LOG_DIR + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + $outpath = $outpath/$CAMERA.$DET_TYPE.$DET_ID + $logfile = $outpath/$CAMERA.$DET_TYPE.norm.$DET_ID.$ITERATION.log + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080109/ippTasks/detrend.process.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/detrend.process.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/detrend.process.pro (revision 16023) @@ -0,0 +1,328 @@ +## this file contains the tasks for running the detrend processing stage +## these tasks use the books detPendingProcessedImfile and detPendingProcessedExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detPendingProcessedImfile +book init detPendingProcessedExp + +macro detproc.reset + book init detPendingProcessedImfile + book init detPendingProcessedExp +end + +macro detproc.status + echo detPendingProcessedImfile + book listbook detPendingProcessedImfile + echo detPendingProcessedExp + book listbook detPendingProcessedExp +end + +macro detproc.on + task detrend.process.load + active true + end + task detrend.process.run + active true + end + task detrend.processexp.load + active true + end + task detrend.processexp.run + active true + end +end + +macro detproc.off + task detrend.process.load + active false + end + task detrend.process.run + active false + end + task detrend.processexp.load + active false + end + task detrend.processexp.run + active false + end +end + + +# these variables will cycle through the known database names +$detPendingProcessedImfile_DB = 0 +$detPendingProcessedExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingProcessedImfile +# compare the new list with the ones already selected +task detrend.process.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detproc.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -toprocessedimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedImfile_DB + command dettool -toprocessedimfile -limit 20 -dbname $DB:$detPendingProcessedImfile_DB + $detPendingProcessedImfile_DB ++ + if ($detPendingProcessedImfile_DB >= $DB:n) set detPendingProcessedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingProcessedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingProcessedImfile + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand + end +end + +# run detrend_process_imfile.ps on pending images +task detrend.process.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingProcessedImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingProcessedImfile + book getpage detPendingProcessedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingProcessedImfile $pageName pantaskState RUN + book getword detPendingProcessedImfile $pageName det_id -var DET_ID + book getword detPendingProcessedImfile $pageName det_type -var DET_TYPE + book getword detPendingProcessedImfile $pageName exp_id -var EXP_ID + book getword detPendingProcessedImfile $pageName class_id -var CLASS_ID + book getword detPendingProcessedImfile $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedImfile $pageName uri -var URI + book getword detPendingProcessedImfile $pageName camera -var CAMERA + book getword detPendingProcessedImfile $pageName workdir -var WORKDIR + book getword detPendingProcessedImfile $pageName dbname -var DBNAME + book getword detPendingProcessedImfile $pageName reduction -var REDUCTION + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + ## output log filename + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detproc.%s" $outpath $EXP_TAG $DET_ID + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` + if ("$logfile" == "") + sprintf logfile "%s.%s.log" $outroot $CLASS_ID + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --det_type $DET_TYPE --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingProcessedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingProcessedImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingProcessedExp +# compare the new list with the ones already selected +task detrend.processexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detproc.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -toprocessedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedExp_DB + command dettool -toprocessedexp -limit 20 -dbname $DB:$detPendingProcessedExp_DB + $detPendingProcessedExp_DB ++ + if ($detPendingProcessedExp_DB >= $DB:n) set detPendingProcessedExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingProcessedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingProcessedExp + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.processexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingProcessedExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new exposures in detPendingProcessedExp + book getpage detPendingProcessedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingProcessedExp $pageName pantaskState RUN + book getword detPendingProcessedExp $pageName det_id -var DET_ID + book getword detPendingProcessedExp $pageName exp_id -var EXP_ID + book getword detPendingProcessedExp $pageName det_type -var DET_TYPE + book getword detPendingProcessedExp $pageName camera -var CAMERA + book getword detPendingProcessedExp $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedExp $pageName workdir -var WORKDIR + book getword detPendingProcessedExp $pageName dbname -var DBNAME + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # XXX push this into the filerules? + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detproc.%s" $outpath $EXP_TAG $DET_ID + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot` + if ("$logfile" == "") + $logfile = $outroot.log + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit detPendingProcessedExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingProcessedExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080109/ippTasks/detrend.reject.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/detrend.reject.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/detrend.reject.pro (revision 16023) @@ -0,0 +1,171 @@ +## this file contains the tasks for running the detrend processing stage +## these tasks use the book detRejectExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detRejectExp + +macro detreject.reset + book init detRejectExp +end + +macro detreject.status + book listbook detRejectExp +end + +macro detreject.on + task detrend.reject.load + active true + end + task detrend.reject.run + active true + end +end + +macro detreject.off + task detrend.reject.load + active false + end + task detrend.reject.run + active false + end +end + +# this variable will cycle through the known database names +$detRejectExp_DB = 0 + +# select images ready for copy +# new entries are added to detRejectExp +# compare the new list with the ones already selected +task detrend.reject.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detrend.reject.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -todetrunsummary -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detRejectExp_DB + command dettool -todetrunsummary -limit 20 -dbname $DB:$detRejectExp_DB + $detRejectExp_DB ++ + if ($detRejectExp_DB >= $DB:n) set detRejectExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detRejectExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detRejectExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detRejectExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand failure + end +end + +# copy new images, sending job to desired host +task detrend.reject.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detRejectExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detRejectExp + book getpage detRejectExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detRejectExp $pageName pantaskState RUN + book getword detRejectExp $pageName det_id -var DET_ID + book getword detRejectExp $pageName iteration -var ITERATION + book getword detRejectExp $pageName det_type -var DET_TYPE + book getword detRejectExp $pageName mode -var MODE + book getword detRejectExp $pageName camera -var CAMERA + book getword detRejectExp $pageName workdir -var WORKDIR + book getword detRejectExp $pageName dbname -var DBNAME + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + if ("$WORKDIR" == "NULL") + $outroot = `pwd` + else + $outroot = `ipp_datapath.pl $WORKDIR` + end + sprintf outroot "%s/%s.%s.%s" $outroot $CAMERA $DET_TYPE $DET_ID + sprintf logfile "%s/%s.%s.%s.%s.detreject.log" $outroot $CAMERA $DET_TYPE $DET_ID $ITERATION + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + $run = detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detRejectExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand failure + book setword detRejectExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080109/ippTasks/detrend.resid.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/detrend.resid.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/detrend.resid.pro (revision 16023) @@ -0,0 +1,328 @@ +## this file contains the tasks for running the detrend processing stage +## these tasks use the books detPendingResidImfile and detPendingResidExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detPendingResidImfile +book init detPendingResidExp + +macro detresid.reset + book init detPendingResidImfile + book init detPendingResidExp +end + +macro detresid.status + book listbook detPendingResidImfile + book listbook detPendingResidExp +end + +macro detresid.on + task detrend.resid.load + active true + end + task detrend.resid.run + active true + end + task detrend.residexp.load + active true + end + task detrend.residexp.run + active true + end +end + +macro detresid.off + task detrend.resid.load + active false + end + task detrend.resid.run + active false + end + task detrend.residexp.load + active false + end + task detrend.residexp.run + active false + end +end + +# these variables will cycle through the known database names +$detPendingResidImfile_DB = 0 +$detPendingResidExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingResidImfile +# compare the new list with the ones already selected +task detrend.resid.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detresid.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -toresidimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingResidImfile_DB + command dettool -toresidimfile -limit 20 -dbname $DB:$detPendingResidImfile_DB + $detPendingResidImfile_DB ++ + if ($detPendingResidImfile_DB >= $DB:n) set detPendingResidImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingResidImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingResidImfile + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.resid.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingResidImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingResidImfile + book getpage detPendingResidImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingResidImfile $pageName pantaskState RUN + book getword detPendingResidImfile $pageName det_id -var DET_ID + book getword detPendingResidImfile $pageName exp_id -var EXP_ID + book getword detPendingResidImfile $pageName iteration -var ITERATION + book getword detPendingResidImfile $pageName det_type -var DET_TYPE + book getword detPendingResidImfile $pageName mode -var MODE + book getword detPendingResidImfile $pageName exp_tag -var EXP_TAG + book getword detPendingResidImfile $pageName class_id -var CLASS_ID + book getword detPendingResidImfile $pageName uri -var URI + book getword detPendingResidImfile $pageName det_uri -var DET_URI + book getword detPendingResidImfile $pageName camera -var CAMERA + book getword detPendingResidImfile $pageName workdir -var WORKDIR + book getword detPendingResidImfile $pageName dbname -var DBNAME + book getword detPendingResidImfile $pageName reduction -var REDUCTION + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # XXX use ipp_filename.pl to lookup output file names + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detresid.%s.%s" $outpath $EXP_TAG $DET_ID $ITERATION + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` + if ("$logfile" == "") + sprintf logfile "%s.%s.log" $outroot $CLASS_ID + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create command + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingResidImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingResidExp +# compare the new list with the ones already selected +task detrend.residexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detresid.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -toresidexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingResidExp_DB + command dettool -toresidexp -limit 20 -dbname $DB:$detPendingResidExp_DB + $detPendingResidExp_DB ++ + if ($detPendingResidExp_DB >= $DB:n) set detPendingResidExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingResidExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingResidExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.residexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingResidExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingResidExp + book getpage detPendingResidExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingResidExp $pageName pantaskState RUN + book getword detPendingResidExp $pageName det_id -var DET_ID + book getword detPendingResidExp $pageName exp_id -var EXP_ID + book getword detPendingResidExp $pageName iteration -var ITERATION + book getword detPendingResidExp $pageName det_type -var DET_TYPE + book getword detPendingResidExp $pageName mode -var MODE + book getword detPendingResidExp $pageName exp_tag -var EXP_TAG + book getword detPendingResidExp $pageName include -var INCLUDE + book getword detPendingResidExp $pageName camera -var CAMERA + book getword detPendingResidExp $pageName workdir -var WORKDIR + book getword detPendingResidExp $pageName dbname -var DBNAME + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + # XXX use ipp_filename.pl to lookup output file names + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detresid.%s.%s" $outpath $EXP_TAG $DET_ID $ITERATION + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id NONE --basename $outroot` + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingResidExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080109/ippTasks/detrend.stack.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/detrend.stack.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/detrend.stack.pro (revision 16023) @@ -0,0 +1,181 @@ +## this file contains the tasks for running the detrend stacking stage +## these tasks use the book detPendingStackedImfile + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detPendingStackedImfile + +macro detstack.reset + book init detPendingStackedImfile +end + +macro detstack.status + echo detPendingStackedImfile + book listbook detPendingStackedImfile +end + +macro detstack.on + task detrend.stack.load + active true + end + task detrend.stack.run + active true + end +end + +macro detstack.off + task detrend.stack.load + active false + end + task detrend.stack.run + active false + end +end + +# this variable will cycle through the known database names +$detPendingStackedImfile_DB = 0 + +# select images ready for detrend_stack.pl +# new entries are added to detPendingStackedImfile +# compare the new list with the ones already selected +task detrend.stack.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detstack.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tostacked -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingStackedImfile_DB + command dettool -tostacked -limit 20 -dbname $DB:$detPendingStackedImfile_DB + $detPendingStackedImfile_DB ++ + if ($detPendingStackedImfile_DB >= $DB:n) set detPendingStackedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingStackedImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingStackedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingStackedImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.stack.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 120 + + task.exec + book npages detPendingStackedImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingStackedImfile + book getpage detPendingStackedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingStackedImfile $pageName pantaskState RUN + book getword detPendingStackedImfile $pageName det_id -var DET_ID + book getword detPendingStackedImfile $pageName iteration -var ITERATION + book getword detPendingStackedImfile $pageName det_type -var DET_TYPE + book getword detPendingStackedImfile $pageName class_id -var CLASS_ID + book getword detPendingStackedImfile $pageName camera -var CAMERA + book getword detPendingStackedImfile $pageName workdir -var WORKDIR + book getword detPendingStackedImfile $pageName dbname -var DBNAME + book getword detPendingStackedImfile $pageName reduction -var REDUCTION + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s" $outpath $CAMERA $DET_TYPE $DET_ID + sprintf outroot "%s/%s.%s.%s.%s" $outpath $CAMERA $DET_TYPE $DET_ID $ITERATION + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` + if ("$logfile" == "") + sprintf logfile "%s.%s.log" $outroot $CLASS_ID + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + # save the pageName for future reference below + options $pageName + + $run = detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingStackedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingStackedImfile $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080109/ippTasks/diff.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/diff.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/diff.pro (revision 16023) @@ -0,0 +1,180 @@ +### This file contains panTasks definitions for performing the image differencing. + +### After a difference (with associated diff_id) is defined, the difference is performed +### (tasks in diffSkyfile). + + +### Setups +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/diff +exec mkdir -p $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init diffSkyfile + +### Database lists +$diffSkycell_DB = 0 + +### Check status of diffing tasks +macro diff.status + book listbook diffSkyfile +end + +### Reset diffing tasks +macro diff.reset + book init diffSkyfile +end + +### Turn diffing tasks on +macro diff.on + task diff.skycell.load + active true + end + task diff.skycell.run + active true + end +end + +### Turn diffing tasks off +macro diff.off + task diff.skycell.load + active false + end + task diff.skycell.run + active false + end +end + + +### Load tasks for doing the differences +### Tasks are loaded into diffSkyfile. +task diff.skycell.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/diff.skycell.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command difftool -todiffskyfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$diffSkycell_DB + command difftool -todiffskyfile -limit 20 -dbname $DB:$diffSkycell_DB + $diffSkycell_DB ++ + if ($diffSkycell_DB >= $DB:n) set diffSkycell_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout diffSkyfile -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffSkyfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffSkyfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the diff overlaps +### Tasks are taken from diffSkyfile. +task diff.skycell.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 60 + + task.exec + book npages diffSkyfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in diffSkyfile (pantaskState == INIT) + book getpage diffSkyfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffSkyfile $pageName pantaskState RUN + book getword diffSkyfile $pageName diff_id -var DIFF_ID + book getword diffSkyfile $pageName workdir -var WORKDIR + book getword diffSkyfile $pageName dbname -var DBNAME + + # specify choice of remote host + # XXX need to choose based on diffs + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + if ("$WORKDIR" == "NULL") + $outroot = `pwd` + else + $outroot = `ipp_datapath.pl $WORKDIR` + end + sprintf outroot "%s/diff.%s" $outroot $DIFF_ID + sprintf logfile "%s/skycell.log" $outroot + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + $run = diff_skycell.pl --diff_id $DIFF_ID + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit diffSkyfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffSkyfile $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branch_20080109/ippTasks/isp.det =================================================================== --- /branches/eam_branch_20080109/ippTasks/isp.det (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/isp.det (revision 16023) @@ -0,0 +1,91 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/bias -det_type bias -select_exp_type bias -select_dateobs_begin 2007-01-13T00:00:00" + dbname STR isp + pantaskState STR DONE + detRunID STR 1 +END + +detRunDef METADATA + detRunName STR DARK-warm + prereq STR BIAS + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/dark -det_type dark -select_exp_type dark -select_exp_time_min 0.5 -select_ccd_temp_min -27.5 -select_ccd_temp_max -24.0 -ccd_temp_min -27.5 -ccd_temp_max -24.0" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK-cool + prereq STR BIAS + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/dark -det_type dark -select_exp_type dark -select_exp_time_min 0.5 -select_ccd_temp_max -27.5 -ccd_temp_max -27.5" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR "DARK-warm DARK-cool" + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/shutter -det_type shutter -select_exp_type skyflat -select_exp_time_min 0.05 -select_filter g -select_dateobs_begin 2007-01-14T04:00:00 -select_dateobs_end 2007-01-14T05:00:00 -select_sat_pixel_frac_max 0.1" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-g + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/g -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter g -filter g -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/r -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter r -filter r -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/i -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter i -filter i -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-z + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/z -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter z -filter z -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-y + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/y -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter y -filter y -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FRINGE-z + prereq STR FLAT-z + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/fringe/z -det_type fringe -select_exp_type object -select_exp_time_min 0.05 -select_filter z -filter z -select_dateobs_begin 2007-01-14T09:19:00 -select_dateobs_end 2007-01-18" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FRINGE-y + prereq STR FLAT-y + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/fringe/y -det_type fringe -select_exp_type object -select_exp_time_min 0.05 -select_filter y -filter y -select_dateobs_begin 2007-01-13 -select_dateobs_end 2007-01-18" + dbname STR isp + pantaskState STR NULL +END Index: /branches/eam_branch_20080109/ippTasks/magic.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/magic.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/magic.pro (revision 16023) @@ -0,0 +1,229 @@ + +### Setups +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/magic +exec mkdir -p $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init magicToProcess +book init magicToMask +book init magicToSkyfileMask + +### Database lists +$magic_DB = 0 + +### Check status of warping tasks +#macro warp.status +# book listbook warpInputExp +# book listbook warpPendingSkyCell +#end + +### Reset warping tasks +#macro warp.reset +# book init warpInputExp +# book init warpPendingSkyCell +#end + +### Turn warping tasks on +# macro warp.on +# task warp.exp.load +# active true +# end +# task warp.exp.run +# active true +# end +# task warp.skycell.load +# active true +# end +# task warp.skycell.run +# active true +# end +# end +# +# ### Turn warping tasks off +# macro warp.off +# task warp.exp.load +# active false +# end +# task warp.exp.run +# active false +# end +# task warp.skycell.load +# active false +# end +# task warp.skycell.run +# active false +# end +# end + +# populate magicToProcess +task magic.toprocess + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/magic.toprocess.log + + task.exec + if ($DB:n == 0) + option DEFAULT + run = command magictool -toprocess -limit 20 + if ($VERBOSE > 1) + echo command $run + end + command $run + else + # save the DB name for the exit tasks + option $DB:$magic_DB + command magictool -toprocess -limit 20 -dbname $DB:$magic_DB + $magic_DB ++ + if ($magic_DB >= $DB:n) set magic_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToProcess -key magic_id:node -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToProcess + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToProcess + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +# populate magicToMask +task magic.tomask + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/magic.tomask.log + + task.exec + if ($DB:n == 0) + option DEFAULT + run = command magictool -tomask -limit 20 + if ($VERBOSE > 1) + echo command $run + end + command $run + else + # save the DB name for the exit tasks + option $DB:$magic_DB + command magictool -tomask -limit 20 -dbname $DB:$magic_DB + $magic_DB ++ + if ($magic_DB >= $DB:n) set magic_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToMask -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToProcess + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToMask + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +# populate magicToMask +task magic.toskyfilemask + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/magic.toskyfilemask.log + + task.exec + if ($DB:n == 0) + option DEFAULT + run = command magictool -toskyfilemask -limit 20 + if ($VERBOSE > 1) + echo command $run + end + command $run + else + # save the DB name for the exit tasks + option $DB:$magic_DB + command magictool -toskyfilemask-limit 20 -dbname $DB:$magic_DB + $magic_DB ++ + if ($magic_DB >= $DB:n) set magic_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToSkyfileMask -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToProcess + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToSkyfileMask + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/eam_branch_20080109/ippTasks/mkdetrend.config =================================================================== --- /branches/eam_branch_20080109/ippTasks/mkdetrend.config (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/mkdetrend.config (revision 16023) @@ -0,0 +1,42 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type BIAS -select_exp_type BIAS" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK + prereq STR BIAS + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR DARK + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i" + dbname STR eamtest + pantaskState STR NULL +END Index: /branches/eam_branch_20080109/ippTasks/mkdetrend.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/mkdetrend.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/mkdetrend.pro (revision 16023) @@ -0,0 +1,217 @@ + +# example output from a dettool -definebyquery +# detRun METADATA +# det_id STR 3 +# iteration S32 0 +# det_type STR SHUTTER +# mode STR master +# state STR run +# filelevel STR FPA +# END + +macro load.detRunDefs + if ($0 != 2) + echo "USAGE: load.detRunDefs (filename)" + break + end + queueload detruns -x "cat $MODULES:0/$1" + ipptool2book detruns detRunDef -key detRunName -uniq -setword pantaskState INIT + queuedelete detruns +end + +task detrun.launch + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + # each outstanding detrend run blocks the next one. only launch the next + # if the current one is done + task.exec + local i j N found missed detRunName prereq prereqList prereqState cmdOptions dbname + + # how many detRunDefs are waiting to be started? + book npages detRunDef -var N -key pantaskState INIT + # echo "detrun.launch N: $N" + if ($N == 0) break + + $found = 0 + + # search the detRunDef book for an entry which is unstarted (state NULL) + # and for which the dependencies are done + for i 0 $N + book getpage detRunDef $i -var detRunName -key pantaskState INIT + # echo "detrun.launch detRunName: $detRunName" + if ("$detRunName" == "NULL") break + + book getword detRunDef $detRunName prereq -var prereqList + if (("$prereqList" == "NULL") || ("$prereqList" == "NONE")) + $found = 1 + last + end + + list prereq -split $prereqList + $missed = 0 + for j 0 $prereq:n + # prereq must exist and be in state of DONE + book getword detRunDef $prereq:$j pantaskState -var prereqState + + # echo "detrun.launch prereqState: $prereqState" + # echo "detrun.launch detRunName: $detRunName" + if ("$prereqState" != "DONE") + $missed = 1 + last + end + end + if ($missed == 0) + $found = 1 + last + end + + if ($found == 0) + if ($VERBOSE >= 2) + echo "$detRunName not ready" + end + else + if ($VERBOSE >= 2) + echo "$detRunName ready" + end + end + end + + if ($found == 0) break + + # echo "detrun.launch detRunName: $detRunName" + + book getword detRunDef $detRunName options -var cmdOptions + book getword detRunDef $detRunName dbname -var dbname + book setword detRunDef $detRunName pantaskState INIT + + if ($VERBOSE >= 1) + echo "starting detrend $detRunName :" + echo "command dettool $cmdOptions -dbname $dbname" + end + + options $dbname $detRunName + command dettool $cmdOptions -dbname $dbname + end + + task.exit $EXIT_SUCCESS + local detRunID + + $dbname = $options:0 + $detRunName = $options:1 + + # convert 'stdout' to book format + ipptool2book stdout detRun -key det_id -uniq + if ($VERBOSE > 2) + book listbook detRun + end + + # we should have launched only a single detRun + book getpage detRun 0 -var detRunID + if ("$detRunID" == "NULL") + echo "ERROR:detRun not defined" + break + end + + # we need to track the detID generated + book setword detRunDef $detRunName detRunID $detRunID + book setword detRunDef $detRunName pantaskState RUN + + # drop the detRun book after we've grabbed the state + book delete detRun + end +end + +$detRunNcheck = 0 + +# loop over the outstanding detRuns and update state +task detrun.check + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + # each outstanding detrend run blocks the next one. only launch the next + # if the current one is done + task.exec + local i N found detRunName detRunID dbname + + book npages detRunDef -var N -key pantaskState RUN + if ($N == 0) break + if ($detRunNcheck >= $N) + $detRunNcheck = 0 + end + + # try the next detRun entry which is in state RUN + book getpage detRunDef $detRunNcheck -var detRunName -key pantaskState RUN + $detRunNcheck ++ + + if ("$detRunName" == "NULL") + if ($VERBOSE >= 3) + echo "no more active detruns (Ncheck = $detRunNcheck)" + end + break + end + + # info needed to check on current state + book getword detRunDef $detRunName detRunID -var detRunID + book getword detRunDef $detRunName dbname -var dbname + + if ($VERBOSE >= 2) + echo " checking on $detRunName" + echo "command dettool -runs -det_id $detRunID -dbname $dbname" + end + + # we need a dettool command which takes a specific det_id and returns + # the current state + options $dbname $detRunName + command dettool -runs -det_id $detRunID -dbname $dbname + end + + task.exit $EXIT_SUCCESS + local detRunID state + + $dbname = $options:0 + $detRunName = $options:1 + + # convert 'stdout' to book format + ipptool2book stdout detRun -key det_id -uniq + if ($VERBOSE > 2) + book listbook detRun + end + + # we should have launched only a single detRun + book getpage detRun 0 -var detRunID + if ("$detRunID" == "NULL") + echo "ERROR:detRun not defined" + break + end + + # what is the current state of the detRun? + book getword detRun $detRunID state -var state + + # use any other state? + if ("$state" == "stop") + # update the state for this detRunName, drop detRun page + book setword detRunDef $detRunName pantaskState DONE + if ($VERBOSE > 1) + echo "$detRunName is done" + end + else + if ($VERBOSE > 1) + echo "$detRunName is still running" + end + end + + # drop this book when completed + book delete detRun + end +end Index: /branches/eam_branch_20080109/ippTasks/notes.txt =================================================================== --- /branches/eam_branch_20080109/ippTasks/notes.txt (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/notes.txt (revision 16023) @@ -0,0 +1,228 @@ + +2007.10.11 + + The calibration suite uses a database table of known/registered DVO + databases to be calibrated. The suite operates on a segment of the + sky in each run, nominally a section ~1h west of the current ST. + + caltool -pendingdbs returns a list of entries (mdc format), one for + each DVO database in the table. + + caltool -databases [-active] : returns the list of dvo databases + caltool -addcalrun : inserts the stats for a single calibration run + caltool -setstate (active/inactive) : modify the state of a dvo + database + caltool -adddatabase : insert a new database + + XXX a potential source of collisions. the books are marked by + unique keys in the db tables, but a single pantasks run may be + addressing multiple databases. IDs are not unique across + databases. we can solve this by including in the key the dbname + + the calibration suite runs the following tasks, in sequences on each + of the databases: + + * resort, relphot, uniphot, relastro.objects, relastro.images + + ** detrend correction: any detrend images which need to be corrected + get inserted in the detCorrectedExp table. + + dettool commands needed: + * add a detrend exposure (det_id, exp_id) to the detCorrectedExp table + * add a detrend imfile (det_id, exp_id, class_id) to the + detCorrectedImfile table + * select pending detCorrectedExp + * select pending detCorrectedImfiles + + +2007.01.08 + + I have updated pantasks to used named tags in named groups to make + life much easier when changing the schema of the db tables. The + function ipptool2book convernts the output from an ippTool query in + the form of a set of metadata structures (saved in stdout) into a + book format. + + dettool -tonormalizedstat DetrendNormStat + dettool -tonormalize DetrendNorm + dettool -tonormalizedexp DetrendNormExp + dettool -toprocessedimfile DetrendProcessImfiles + dettool -toprocessedexp DetrendProcessExposures + dettool -residdetrun DetrendRejectExp + dettool -toresidimfile DetrendResidImfiles + dettool -toresidexp DetrendResidExposures + dettool -tostacked DetrendStackClass + p0tool -pendingimfile Phase0Imfiles + p0tool -pendingexp Phase0Exposures + p2tool -pendingimfile Phase2Imfiles + p3tool -pendingexp Phase3Exposures + +2006.08.28 + +- ipptools need to use expID:camera:telescope as unique key (ie, not expID alone) +- p0tool -pendingimfile -simple needs to report the expKey (ie, not expID) +- + + +This directory contains the PanTasks scripts which define the steps +performed by the IPP. These scripts use the ippTools to decide what +images need to be processed, and to update the Metadata Database with +the current state of the analysis. The analysis is performed using +IPP programs which are distributed to the cluster computers as needed. + +1. The Detrend Creation Scripts + +The detrend creation process is among the more complex elements of the +IPP. There are N major steps which must be performed: + +1.1. Define New Detrend Runs + +The script 'detrend.mkruns.pro' defines tasks which initiate new +Detrend Runs. There is a separate task for each camera automatically +processed by the system (mkdetruns.CAMERA). These tasks are executed +on a regular, but infrequent basis. The tasks call external helper +programs which actually run the command dettools -define. The helper +programs are thus responsible for identifying all of the types of +detrend images to be run for a given camera, as well as the correct +set of filters to apply. These tasks are run locally and only affect +the database. PanTasks does not track the detrend runs which are +created. + +- we may need to define these within the PanTasks scripts +- how frequently are these tasks run (1/day?) +- if no raw images are selected by the query, no new detrun should be + created. +- we may need different schedules for the dark (bias/dark) and the + light (flat/fringe) data. And yet another schedule for sky data + (skyflat/skyfringe). + +1.2. Process new raw imfiles + +The script 'detrend.process.pro' defines tasks which processes the new +raw imfiles for any detrend run. This script defines two tasks, +'dettool.raw.load' and 'dettool.raw.process'. The first script +occasionally polls the Metadata Database to select raw imfiles which +have been identified as part of a detrend run. The identified raw +imfiles are added to an internal queue, DetrendImfilesToProcess. Any +raw imfile already in the list is ignored. The imfiles are added to +the queue with a state of NEW. The second task examines the queue for +imfiles with state NEW and defines a ppImage analysis job for that +imfile. The job is sent to pcontrol for processing on the distributed +cluster. At this point, the corresponding queue entry is set to the +state RUN. Upon success, the queue entry is set to DONE and the +database is notified of the successful operation. The DONE entries +are occasionally pruned from the queue; this must be done in the same +task as the initial query and the update of the list based on the +database results. + +- we need to define the columns in the output of dettool -raw +- we need a method to associate an imfile with a node in the cluster +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs +- we need a method to identify the correct recipe (function of type?) + +1.3. Stack new processed imfiles + +The script 'detrend.stack.pro' defines tasks which stack the newly +processed imfiles for any detrend run iteration. This script defines +two tasks, 'dettool.stack.load and 'dettool.stack.process. The first +script occasionally polls the Metadata Database to select class IDs +for detrend runs for which the input imfiles have all been processed. +The identified detRun/classID combination is added to an internal +queue, DetrendClassIDtoStack. The entries are added to the queue with +a state of NEW. The second task examines the queue for entries with a +state of NEW. It defines a ppMerge analysis job for that +detID/classID. The helper script selects the corresponding input +imfiles and passes them to ppMerge. The job is sent to pcontrol for +processing on the distributed cluster. At this point, the +corresponding queue entry is set to the state RUN. Upon success, the +queue entry is set to DONE and the database is notified of the +successful operation. The DONE entries are occasionally pruned from +the queue; this must be done in the same task as the initial query and +the update of the list based on the database results. + +- we need to define the columns in the output of dettool -chip -processed +- we need a method to associate a classID with a node in the cluster +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) + + +1.4. Normalize stacked masters + +The script 'detrend.norm.pro' defines tasks which normalize the newly +processed stacks for any detrend run iteration. This script defines +two tasks, 'dettool.norm.load and 'dettool.norm.process. The first +script occasionally polls the Metadata Database to select detrun IDs +for which the classes have all been stacked. The identified +detRunID/iteration combination is added to an internal queue, +DetrendRunToNormalize. The entries are added to the queue with a +state of NEW. The second task examines the queue for entries with a +state of NEW. It defines a ppNorm analysis job for that +detID/iteration. The job is sent to pcontrol for processing on the +distributed cluster. At this point, the corresponding queue entry is +set to the state RUN. Upon success, the queue entry is set to DONE and +the database is notified of the successful operation. The DONE +entries are occasionally pruned from the queue; this must be done in +the same task as the initial query and the update of the list based on +the database results. + +- we need to define the columns in the output of dettool +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) +- we need to define the ppNorm operation inputs and outputs better + + +1.5. Generate Resid Imfiles + +The script 'detrend.resid.pro' defines tasks which apply the newly +generated detrend image to the corresponding input images to generate +the residual images. This script defines two tasks, +'dettool.resid.load and 'dettool.resid.process. The first script +occasionally polls the Metadata Database to select imfiles for which +the master detrend images has been normalized. The identified imfiles +are added to an internal queue, DetrendResidImfiles. The entries are +added to the queue with a state of NEW. The second task examines the +queue for entries with a state of NEW. It defines a ppImage analysis +job for that imfile. The job is sent to pcontrol for processing on +the distributed cluster. At this point, the corresponding queue entry +is set to the state RUN. Upon success, the queue entry is set to DONE +and the database is notified of the successful operation. The DONE +entries are occasionally pruned from the queue; this must be done in +the same task as the initial query and the update of the list based on +the database results. + +- we need to define the columns in the output of dettool +- can we select the imfiles here without apriori knowledge that the + master has been normalized? +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) +- we need a method to pass the matching detrend image to ppImage in + the correct context (ie, a bias needs to be supplied to the -bias element...) + +1.6. Assess Detrend Results + +The script 'detrend.assess.pro' defines tasks which assess the +residual statistics for completed detrend runs. This script defines +two tasks, 'dettool.assess.load and 'dettool.assess.process. The +first script occasionally polls the Metadata Database to select +detrunIDs/iterations for which the resid imfiles have all been +generated. The identified detruns are added to an internal queue, +DetrendRunToAssess. The entries are added to the queue with a state +of NEW. The second task examines the queue for entries with a state +of NEW. It defines an analysis job for that collection. The job is +sent to pcontrol for processing on the distributed cluster. At this +point, the corresponding queue entry is set to the state RUN. Upon +success, the queue entry is set to DONE and the database is notified +of the successful operation. The DONE entries are occasionally pruned +from the queue; this must be done in the same task as the initial +query and the update of the list based on the database results. + +- the assess tool needs to trigger a new run, if needed. +- what is the command to run on the resid statistics? +- we need to define the columns in the output of dettool +- can we select the detrend runs here? +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) Index: /branches/eam_branch_20080109/ippTasks/pantasks.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/pantasks.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/pantasks.pro (revision 16023) @@ -0,0 +1,340 @@ + +# XXX make these upper-case +$network = 1 +$parallel = 1 +$VERBOSE = 1 +$LOGDIR = `pwd` +$scripts = . + +$loadpoll = 0.25 +$loadexec = 5 +$runpoll = 0.5 +$runexec = 1 + +$EXIT_SUCCESS = 0 +$EXIT_UNKNOWN_ERR = 1 +$EXIT_SYS_ERR = 2 +$EXIT_CONFIG_ERR = 3 +$EXIT_PROG_ERR = 4 +$EXIT_DATA_ERR = 5 + +# DB lists the database names in use; the default ipprc database +# is only used if DB:n is 0 +$DB:n = 0 + +# user function to add databases by name (avoids duplicates) +macro add.database + if ($0 != 2) + echo "USAGE: add.database (db)" + break + end + if ($?DB:n == 0) + list DB -add $1 + return + end + + $found = 0 + for i 0 $DB:n + if ($DB:$i == $1) + $found = 1 + echo "$DB:$i set" + last + end + end + + if ($found == 0) + list DB -add $1 + end +end + +macro del.database + if ($0 != 2) + echo "USAGE: del.database (db)" + break + end + if ($?DB:n == 0) + return + end + +### XXX There is a bug in the below "list DB -del $1" line. + echo "This function doesn't work" +# list DB -del $1 +end + +macro init.cluster.po + $parallel = 1 + controller exit true + + # po05 -- using for pantasks + # po23 -- broken perl + + controller host add po02 + controller host add po03 + controller host add po04 +# controller host add po05 + controller host add po06 + controller host add po07 +# controller host add po08 +# controller host add po09 +# controller host add po10 +# controller host add po11 +# controller host add po12 +# controller host add po13 +# controller host add po14 +# controller host add po15 +# controller host add po16 +# controller host add po17 +# controller host add po18 +# controller host add po19 +# controller host add po20 +# controller host add po21 +# controller host add po22 +# controller host add po23 +# controller host add po24 +end + +macro init.cluster.sn + $parallel = 1 + controller exit true + controller host add sn2 + controller host add sn3 + controller host add sn4 + controller host add sn5 +end + +macro init.isp + list DB -add isp +end + +macro init.essence + list DB -add essence_v2 +end + +macro init.simtest.local + list DB -add simtest + $parallel = 0 + controller exit true +end + +macro init.simtest.default.db + $host = `hostname` + if ($parallel) + controller exit true + controller host add $host + end +end + +macro init.simtest + list DB -add simtest + + $host = `hostname` + if ($parallel) + controller exit true + controller host add $host + end +end + +# need the ability to activate or deactivate specific tasks... + +macro detrend.on + detproc.on + detstack.on + detnorm.on + detresid.on + detreject.on +end + +macro detrend.off + detproc.off + detstack.off + detnorm.off + detresid.off + detreject.off +end + +macro detrend.reset + detproc.reset + detstack.reset + detnorm.reset + detresid.reset + detreject.reset +end + +macro all.on + register.on + detrend.on + chip.on + camera.on + warp.on + diff.on + stack.on +end + +macro all.off + register.off + detrend.off + chip.off + camera.off + warp.off + diff.off + stack.off +end + +macro module.tasks + module register.pro + module chip.pro + module camera.pro + module detrend.process.pro + module detrend.stack.pro + module detrend.norm.pro + module detrend.resid.pro + module detrend.reject.pro + module warp.pro + module diff.pro + module stack.pro +end + +macro showcommand + local n + + if ($VERBOSE) + $command = $taskarg:0 + for n 1 $taskarg:n + $command = $command $taskarg:$n + end + if ($0 == 2) + echo "" + echo "$1 for: $command" + echo "job exit status: $JOB_STATUS" + echo "job host: $JOB_HOSTNAME" + echo "job dtime: $JOB_DTIME" + else + echo "command: $command" + end + end +end + +macro add_standard_args + if ($0 != 2) + echo "Must pass in the command of interest, and this function will supplement" + stop + end + + local command + $command = $$1 + + if ($?DBNAME && "$DBNAME" != "DEFAULT") + $command = $command --dbname $DBNAME + end + if ($?NOOP != 0) + if ($NOOP != 0) + $command = $command --no-op + end + end + if ($?NOUPDATE != 0) + if ($NOUPDATE != 0) + $command = $command --no-update + end + end + $$1 = $command +end + + +macro process_exit + if ($0 != 4) + echo "USAGE: process_exit (bookName) (pageName) (exitCode)" + break + end + + $bookName = $1 + $pageName = $2 + $exitCode = $3 + + if ($VERBOSE > 4) + echo "*** exit status ***" + echo "JOB_STATUS: $JOB_STATUS" + echo "JOB_HOSTNAME: $JOB_HOSTNAME" + echo "JOB_DTIME: $JOB_DTIME" + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + + # success + if ($exitCode == $EXIT_SUCCESS) + # the handler scripts update DB the tables; here we just update the page + book setword $bookName $pageName pantaskState DONE + return + end + + # failure related to the data files + # jobs which result in DATAERR must have db pantaskState updated + if ($exitCode == $EXIT_DATA_ERR) + showcommand failure + book setword $bookName $pageName pantaskState DATA_ERR + return + end + + if ($VERBOSE) + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + + # failure related to the data files + if ($exitCode == $EXIT_SYS_ERR) + # stop + showcommand "system failure" + book setword $bookName $pageName pantaskState SYS_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_CONFIG_ERR) + # stop + showcommand "config error" + book setword $bookName $pageName pantaskState CONFIG_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_PROG_ERR) + # stop + showcommand "programming error" + book setword $bookName $pageName pantaskState PROG_ERR + return + end + + # any other exit status + showcommand failure + book setword $bookName $pageName pantaskState UNKNOWN_ERR +end + +## XXX for the moment, remove all errors +macro process_cleanup + if ($0 != 2) + echo "USAGE: process_cleanup (bookname)" + break + end + + book delpage $1 -key pantaskState DONE + book delpage $1 -key pantaskState SYS_ERR + book delpage $1 -key pantaskState DATA_ERR + book delpage $1 -key pantaskState PROG_ERR + book delpage $1 -key pantaskState CONFIG_ERR + book delpage $1 -key pantaskState UNKNOWN_ERR +end + +macro set.verbosity + if ($0 != 2) + echo "USAGE: set.verbosity (level)" + break + end + $VERBOSE = $1 +end + +macro show.variables + ?? +end Index: /branches/eam_branch_20080109/ippTasks/register.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/register.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/register.pro (revision 16023) @@ -0,0 +1,301 @@ +## this file contains the tasks for running the registration stage +## these tasks use the books regPendingImfile and regPendingExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/register +exec mkdir -p $LOGSUBDIR + +book init regPendingImfile +book init regPendingExp + +macro register.reset + book init regPendingImfile + book init regPendingExp +end + +macro register.status + book listbook regPendingImfile + book listbook regPendingExp +end + +macro register.on + task register.imfile.load + active true + end + task register.imfile.run + active true + end + task register.exp.load + active true + end + task register.exp.run + active true + end +end + +macro register.off + task register.imfile.load + active false + end + task register.imfile.run + active false + end + task register.exp.load + active false + end + task register.exp.run + active false + end +end + +# these variables will cycle through the known database names +$regPendingImfile_DB = 0 +$regPendingExp_DB = 0 + +# select images ready for register analysis +# new entries are added to regPendingImfile +# compare the new list with the ones already selected +task register.imfile.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/register.imfile.log + + # select entries from the current DB; cycle to the next DB, if it exists + # iff the DB list is not set, use the value defined in .ipprc + task.exec + if ($DB:n == 0) + option DEFAULT + command regtool -pendingimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$regPendingImfile_DB + command regtool -pendingimfile -limit 20 -dbname $DB:$regPendingImfile_DB + $regPendingImfile_DB ++ + if ($regPendingImfile_DB >= $DB:n) set regPendingImfile_DB = 0 + end + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingImfile -key exp_id:tmp_class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook regPendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup regPendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the register_imfile.pl script on pending images +task register.imfile.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages regPendingImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in regPendingImfile + book getpage regPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingImfile $pageName pantaskState RUN + + book getword regPendingImfile $pageName exp_id -var EXP_ID + book getword regPendingImfile $pageName tmp_exp_name -var TMP_EXP_NAME + book getword regPendingImfile $pageName tmp_class_id -var TMP_CLASS_ID + book getword regPendingImfile $pageName uri -var URI + book getword regPendingImfile $pageName workdir -var WORKDIR + book getword regPendingImfile $pageName dbname -var DBNAME + + # EXP_TAG is used to generate the unique, but human-readable, filenames + sprintf EXP_TAG "%s.%s" $TMP_EXP_NAME $EXP_ID + + # specify choice of remote host + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule XXX put this in a function? + $outroot = `ipp_datapath.pl $WORKDIR` + sprintf outroot "%s/%s" $outroot $EXP_TAG + sprintf logfile "%s/%s.reg.log" $outroot $TMP_CLASS_ID + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + # save the pageName for future reference below + options $pageName + + $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit regPendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword regPendingImfile $options:0 pantaskState TIMEOUT + end +end + +# select exposures ready for register_exp.pl +task register.exp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/register.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command regtool -pendingexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$regPendingExp_DB + command regtool -pendingexp -limit 20 -dbname $DB:$regPendingExp_DB + $regPendingExp_DB ++ + if ($regPendingExp_DB >= $DB:n) set regPendingExp_DB = 0 + end + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingExp -key exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook regPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup regPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the register_exp.pl script on pending exposures +task register.exp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages regPendingExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in regPendingExp (pantaskState == INIT) + book getpage regPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingExp $pageName pantaskState RUN + book getword regPendingExp $pageName exp_id -var EXP_ID + book getword regPendingExp $pageName tmp_exp_name -var TMP_EXP_NAME + book getword regPendingExp $pageName workdir -var WORKDIR + book getword regPendingExp $pageName dbname -var DBNAME + + # EXP_TAG is used to generate the unique, but human-readable, filenames + sprintf EXP_TAG "%s.%s" $TMP_EXP_NAME $EXP_ID + + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + $outroot = `ipp_datapath.pl $WORKDIR` + sprintf outroot "%s/%s" $outroot $EXP_TAG + sprintf logfile "%s/%s.reg.log" $outroot $EXP_TAG + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + # save the pageName for future reference below + options $pageName + + $run = register_exp.pl --exp_id $EXP_ID --exp_tag $EXP_TAG + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit regPendingExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword regPendingExp $options:0 pantaskState TIMEOUT + end +end + +## XXX add a global path to output files Index: /branches/eam_branch_20080109/ippTasks/simtest.det =================================================================== --- /branches/eam_branch_20080109/ippTasks/simtest.det (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/simtest.det (revision 16023) @@ -0,0 +1,42 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type BIAS -select_exp_type BIAS" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK + prereq STR BIAS + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR DARK + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i" + dbname STR eamtest + pantaskState STR NULL +END Index: /branches/eam_branch_20080109/ippTasks/stack.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/stack.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/stack.pro (revision 16023) @@ -0,0 +1,180 @@ +### This file contains panTasks definitions for performing the image stacking. + +### After a stack (with associated stack_id) is defined, the stack is performed +### (tasks in stackSumSkyfile). + + +### Setups +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/stack +exec mkdir -p $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init stackSumSkyfile + +### Database lists +$stackSkycell_DB = 0 + +### Check status of stacking tasks +macro stack.status + book listbook stackSumSkyfile +end + +### Reset stacking tasks +macro stack.reset + book init stackSumSkyfile +end + +### Turn stacking tasks on +macro stack.on + task stack.skycell.load + active true + end + task stack.skycell.run + active true + end +end + +### Turn stacking tasks off +macro stack.off + task stack.skycell.load + active false + end + task stack.skycell.run + active false + end +end + + +### Load tasks for doing the stack +### Tasks are loaded into stackSumSkyfile. +task stack.skycell.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/stack.skycell.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command stacktool -tosum -limit 20 + else + # save the DB name for the exit tasks + option $DB:$stackSkycell_DB + command stacktool -tosum -limit 20 -dbname $DB:$stackSkycell_DB + $stackSkycell_DB ++ + if ($stackSkycell_DB >= $DB:n) set stackSkycell_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout stackSumSkyfile -key stack_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook stackSumSkyfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup stackSumSkyfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the stack overlaps +### Tasks are taken from stackSumSkyfile. +task stack.skycell.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 60 + + task.exec + book npages stackSumSkyfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in stackSumSkyfile (pantaskState == INIT) + book getpage stackSumSkyfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword stackSumSkyfile $pageName pantaskState RUN + book getword stackSumSkyfile $pageName stack_id -var STACK_ID + book getword stackSumSkyfile $pageName workdir -var WORKDIR + book getword stackSumSkyfile $pageName dbname -var DBNAME + + # specify choice of remote host + # XXX need to choose based on stacks + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + if ("$WORKDIR" == "NULL") + $outroot = `pwd` + else + $outroot = `ipp_datapath.pl $WORKDIR` + end + sprintf outroot "%s/stack.%s" $outroot $STACK_ID + sprintf logfile "%s/skycell.log" $outroot + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + $run = stack_skycell.pl --stack_id $STACK_ID + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit stackSumSkyfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword stackSumSkyfile $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branch_20080109/ippTasks/summit.copy.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/summit.copy.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/summit.copy.pro (revision 16023) @@ -0,0 +1,319 @@ +## PanTasks scripts for Summit Copy + +module pantasks.pro + +# pztool -adddatastore -inst isp -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/skyprobe/index.txt +# pztool -adddatastore -inst gpc1 -telescope ps1 -uri http://conductor/ds/gpc1/index.txt +# pztool -adddatastore -inst allskycam -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/allskycam/index.txt + +$workdir = /data/ipp010.0/ + +$VERBOSE = 1 + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + +# list of DataStores to pull data from +book init pzDataStore +# list of summit exps that need to be queried +book init pzPendingExp +# list of summit imfiles that need to be downloaded +book init pzPendingImfile + +# build a book of datastores to poll for data +task pztool.datastore + command pztool -datastore + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 15 + periods -poll 1 + periods -timeout 15 + + # success + task.exit 0 + # flush pzDataStore book + book init pzDataStore + # convert 'stdout' to book format + ipptool2book stdout pzDataStore -key camera:telescope -uniq + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +$datastore_index = 0 + +# run pzgetexp periodically to populate pzPendingExp in the database (no I/O) +task pzgetexp + # timeout shorter than exec so jobs do not build up + periods -exec 5 + periods -poll 1 + periods -timeout 5 + host local + + task.exec + # find an exp that needs imfiles fetched + book getpage pzDataStore $datastore_index -var pageName + if ("$pageName" == "NULL") break + + # increment our pzDataStore index and loop back to 0 and the end of the + # book + $datastore_index ++ + book npages pzDataStore -var npages + if ($datastore_index == $npages ) + $datastore_index = 0 + end + + book getword pzDataStore $pageName camera -var CAMERA + book getword pzDataStore $pageName telescope -var TELESCOPE + book getword pzDataStore $pageName uri -var URI + + # store the current page + options $pageName + + $run = pzgetexp -uri $URI -inst $CAMERA -telescope $TELESCOPE + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + + command $run + end + + task.exit 0 + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +# build a book of exps/filesetids that need to be queried +task pztool.pendingexp + command pztool -pendingexp -limit 25 + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 15 + periods -poll 1 + periods -timeout 15 + + # success + task.exit 0 + # convert 'stdout' to book format +# ipptool2book stdout pzPendingExp -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT + +# XXX we probably want the database to be user-defined... + ipptool2book stdout pzPendingExp -key exp_name:camera:telescope -uniq -setword pantaskState INIT + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingExp + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +# run pzgetimfiles on pending exps +task pzgetimfile + periods -exec 0.05 + periods -poll 0.025 + periods -timeout 15 + host local + npending 20 + + task.exec + if ($network == 0) break + + # if we are waiting on data, make the interval long + book npages pzPendingExp -var N + if ($N == 0) + periods -exec 20 + break + end + periods -exec 0.05 + + # find an exp that needs imfiles fetched + book getpage pzPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingExp $pageName pantaskState RUN + + book getword pzPendingExp $pageName exp_name -var EXP_NAME + book getword pzPendingExp $pageName camera -var CAMERA + book getword pzPendingExp $pageName telescope -var TELESCOPE + book getword pzPendingExp $pageName dateobs -var DATEOBS + book getword pzPendingExp $pageName exp_type -var EXP_TYPE + book getword pzPendingExp $pageName uri -var URI + book getword pzPendingExp $pageName imfiles -var IMFILES + + # store the current page + options $pageName + + $batman = $EXP_NAME + $run = pzgetimfiles -uri $URI -filesetid $batman -inst $CAMERA -telescope $TELESCOPE + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit 0 + process_exit pzPendingExp $options:0 $JOB_STATUS + end + + task.exit default + showcommand failure + process_exit pzPendingExp $options:0 $JOB_STATUS + end + + task.exit timeout + showcommand timeout + book setword pzPendingExp $options:0 pantaskState TIMEOUT + end +end + + +# build a book of imfiles/files that need to be downloaded +task pztool.pendingimfile + command pztool -pendingimfile -limit 120 + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 5 + periods -poll 1 + periods -timeout 5 + + # success + task.exit 0 + # convert 'stdout' to book format +# ipptool2book stdout pzPendingImfile -key exp_name:camera:telescope:class:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + + ipptool2book stdout pzPendingImfile -key exp_name:camera:telescope:class:class_id -uniq -setword pantaskState INIT + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingImfile + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + + +# get dsget to retreive an imfile and then call pztool -copydone +task dsget + periods -exec 5 + periods -poll 0.05 + periods -timeout 120 + host anyhost + + task.exec + if ($network == 0) break + + # if we are waiting on data, make the interval long + book npages pzPendingImfile -var N + if ($N == 0) + periods -exec 20 + break + end + periods -exec 0.05 + + # find an exp that needs imfiles fetched + book getpage pzPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingImfile $pageName pantaskState RUN + + book getword pzPendingImfile $pageName uri -var URI + book getword pzPendingImfile $pageName exp_name -var EXP_NAME + book getword pzPendingImfile $pageName class_id -var CLASS_ID + book getword pzPendingImfile $pageName bytes -var BYTES + book getword pzPendingImfile $pageName md5sum -var MD5SUM + book getword pzPendingImfile $pageName dateobs -var DATEOBS + book getword pzPendingImfile $pageName camera -var $CAMERA + + # 2007-08-30T05:09:59Z + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 5 2 MONTH + substr $DATEOBS 8 2 DAY + + # figure out filename + $FILENAME = $workdir/$CAMERA/$YEAR/$MONTH/$DAY/$EXP_NAME/$EXP_NAME.$CLASS_ID.fits + + book setword pzPendingImfile $pageName filename $FILENAME + + # store the pageName with this job + options $pageName + + if ($MD5SUM == 0) + $run = dsget --uri $URI --filename $FILENAME --bytes $BYTES + else + $run = dsget --uri $URI --filename $FILENAME --bytes $BYTES --md5 $MD5SUM + end + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit 0 + # get related info from the associated page + book getword pzPendingImfile $options:0 exp_name -var EXP_NAME + book getword pzPendingImfile $options:0 class -var CLASS + book getword pzPendingImfile $options:0 camera -var CAMERA + book getword pzPendingImfile $options:0 telescope -var TELESCOPE + book getword pzPendingImfile $options:0 class_id -var CLASS_ID + book getword pzPendingImfile $options:0 filename -var FILENAME + + # update the database with the success + $run = pztool -copydone -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -class $CLASS -class_id $CLASS_ID -uri $FILENAME + + # create the command line + if ($VERBOSE > 1) + echo exec $run + end + exec $run + + # update the pantasks process tables + process_exit pzPendingImfile $options:0 $JOB_STATUS + end + + task.exit default + showcommand failure + process_exit pzPendingImfile $options:0 $JOB_STATUS + end + + task.exit timeout + showcommand timeout + book setword pzPendingImfile $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080109/ippTasks/testscript.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/testscript.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/testscript.pro (revision 16023) @@ -0,0 +1,80 @@ +# test for pcontrol +macro test.pcontrol.simple + stop + for i 0 100 + job ls -R /etc + end + + status + host add alala + + status + run +end + +# test for pcontrol +macro test.pcontrol + exec p0tool -pendingimfile -simple | head -n 100 > list.dat + + stop + list line -x "cat list.dat" + echo $line:0 + for i 0 $line:n + list word -split $line:$i + job phase0_imfile.pl --exp_tag $word:0 --class_id fpa --uri $word:3 + end + + status + host add sn2 + host add sn3 + host add sn4 + host add sn5 + + status + run +end + +# test for pclient +macro test.pclient + exec p0tool -pendingimfile -simple | head -n 100 > list.dat + + list line -x "cat list.dat" + echo $line:0 + for i 0 $line:n + list word -split $line:$i + job phase0_imfile.pl --exp_tag $word:0 --class_id fpa --uri $word:3 + + for i 0 50 + check + output /dev/null + status + output stdout + if ($JOBSTATUS == EXIT) + $i = 50 + reset + echo "DONE" + else + usleep 200000 + end + end + if ($JOBSTATUS != EXIT) + echo TIMEOUT*** + end + end +end + +macro simple.test + for j 0 100 + job ls + for i 0 50 + check + status + if ($JOBSTATUS == EXIT) + $i = 50 + reset + else + usleep 10000 + end + end + end +end Index: /branches/eam_branch_20080109/ippTasks/warp.pro =================================================================== --- /branches/eam_branch_20080109/ippTasks/warp.pro (revision 16023) +++ /branches/eam_branch_20080109/ippTasks/warp.pro (revision 16023) @@ -0,0 +1,319 @@ +### This file contains panTasks definitions for performing the image warping. + +### This is done in two (main) steps. After a warp (with associated warp_id) is defined, +### overlaps between the exposure being warped and skycells are calculated (tasks in warpInputExp). +### Then for each skycell, the warp is made (tasks in warpPendingSkycell). + + +### Setups +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/warp +exec mkdir -p $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init warpInputExp +book init warpPendingSkyCell + +### Database lists +$warpExp_DB = 0 +$warpSkycell_DB = 0 + +### Check status of warping tasks +macro warp.status + book listbook warpInputExp + book listbook warpPendingSkyCell +end + +### Reset warping tasks +macro warp.reset + book init warpInputExp + book init warpPendingSkyCell +end + +### Turn warping tasks on +macro warp.on + task warp.exp.load + active true + end + task warp.exp.run + active true + end + task warp.skycell.load + active true + end + task warp.skycell.run + active true + end +end + +### Turn warping tasks off +macro warp.off + task warp.exp.load + active false + end + task warp.exp.run + active false + end + task warp.skycell.load + active false + end + task warp.skycell.run + active false + end +end + +### Load tasks for calculating the warp overlaps +### Tasks are loaded into warpInputExp. +task warp.exp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/warp.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command warptool -tooverlap -limit 20 + else + # save the DB name for the exit tasks + option $DB:$warpExp_DB + command warptool -tooverlap -limit 20 -dbname $DB:$warpExp_DB + $warpExp_DB ++ + if ($warpExp_DB >= $DB:n) set warpExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpInputExp -key warp_id:cam_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpInputExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpInputExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpInputExp. +task warp.exp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 60 + + task.exec + book npages warpInputExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in warpInputExp (pantaskState == INIT) + book getpage warpInputExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpInputExp $pageName pantaskState RUN + book getword warpInputExp $pageName warp_id -var WARP_ID + book getword warpInputExp $pageName camera -var CAMERA + book getword warpInputExp $pageName workdir -var WORKDIR + book getword warpInputExp $pageName dbname -var DBNAME + + # specify choice of remote host + # XXX need to choose based on warps + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + if ("$WORKDIR" == "NULL") + $outroot = `pwd` + else + $outroot = `ipp_datapath.pl $WORKDIR` + end + sprintf outroot "%s/warp.%s" $outroot $WARP_ID + sprintf logfile "%s/overlap.log" $outroot + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + $run = warp_overlap.pl --warp_id $WARP_ID --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit warpInputExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpInputExp $options:0 pantaskState TIMEOUT + end +end + + +### Load tasks for doing the warps +### Tasks are loaded into warpPendingSkyCell. +task warp.skycell.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/warp.skycell.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command warptool -towarped -limit 20 + else + # save the DB name for the exit tasks + option $DB:$warpSkycell_DB + command warptool -towarped -limit 20 -dbname $DB:$warpSkycell_DB + $warpSkycell_DB ++ + if ($warpSkycell_DB >= $DB:n) set warpSkycell_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpPendingSkyCell -key warp_id:skycell_id:tess_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpPendingSkyCell + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpPendingSkyCell + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpPendingSkyCell. +task warp.skycell.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 60 + + task.exec + book npages warpPendingSkyCell -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in warpPendingSkyCell (pantaskState == INIT) + book getpage warpPendingSkyCell 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpPendingSkyCell $pageName pantaskState RUN + book getword warpPendingSkyCell $pageName warp_id -var WARP_ID + book getword warpPendingSkyCell $pageName skycell_id -var SKYCELL_ID + book getword warpPendingSkyCell $pageName tess_id -var TESS_ID + book getword warpPendingSkyCell $pageName camera -var CAMERA + book getword warpPendingSkyCell $pageName workdir -var WORKDIR + book getword warpPendingSkyCell $pageName dbname -var DBNAME + + # specify choice of remote host + # XXX need to choose based on warps + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + if ("$WORKDIR" == "NULL") + $outroot = `pwd` + else + $outroot = `ipp_datapath.pl $WORKDIR` + end + sprintf outroot "%s/warp.%s" $outroot $WARP_ID + sprintf logfile "%s/skycell.log" $outroot + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + $run = warp_skycell.pl --warp_id $WARP_ID --skycell_id $SKYCELL_ID --tess_id $TESS_ID --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit warpPendingSkyCell $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpPendingSkyCell $options:0 pantaskState TIMEOUT + end +end +