Index: trunk/ippTasks/Makefile.am =================================================================== --- trunk/ippTasks/Makefile.am (revision 42098) +++ trunk/ippTasks/Makefile.am (revision 42385) @@ -50,4 +50,5 @@ bg.regeneration.pro \ remote.pro \ + fpcamera.pro \ fullforce.pro Index: trunk/ippTasks/automate.pro =================================================================== --- trunk/ippTasks/automate.pro (revision 42098) +++ trunk/ippTasks/automate.pro (revision 42385) @@ -17,4 +17,19 @@ # DONE (action finished) +macro automate.status + + ## Pull out the ones that are to be run regularly + local npages + book npages automate -var npages + for i 0 $npages + book getpage automate $i -var pageName + echo $i : $pageName + book getword automate $pageName pantaskState -var myState + if ("$myState" != "NULL") + echo $pageName : $myState + end + end +end + macro automate.reset ## probably should not always init @@ -27,7 +42,15 @@ break end - queueload tmp -x "cat $MODULES:0/$1" + + file $1 isFound + if ($isFound) + queueload tmp -x "cat $1" + else + # search module path list for existing file + queueload tmp -x "cat $MODULES:0/$1" + end pwd -var cwd + $username = `whoami` ## interpolate standard values @@ -35,6 +58,4 @@ queuesubstr tmp @DBNAME@ $3 queuesubstr tmp @CWD@ $cwd - - $username = `whoami` queuesubstr tmp @USER@ $username Index: trunk/ippTasks/fpcamera.pro =================================================================== --- trunk/ippTasks/fpcamera.pro (revision 42385) +++ trunk/ippTasks/fpcamera.pro (revision 42385) @@ -0,0 +1,268 @@ +## fpcamera.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the forced photometry camera analysis stage +## these tasks use the book fpcamPendingExp + +# test for required global variables +check.globals + +# fpcamera.pro should have a more restricted polling limit (to avoid stress with getstar) +if ($?POLL_LIMIT_FPCAMERA == 0) set POLL_LIMIT_FPCAMERA = 10 + +macro set.fpcamera.poll + if ($0 != 2) + echo "USAGE:set.fpcamera.poll (value)" + break + end + + $POLL_LIMIT_FPCAMERA = $1 +end + +macro get.fpcamera.poll + echo "fpcamera poll limit: $POLL_LIMIT_FPCAMERA" +end + +book init fpcamPendingExp + +macro fpcamera.status + book listbook fpcamPendingExp +end + +macro fpcamera.reset + book init fpcamPendingExp +end + +macro fpcamera.on + task fpcamera.exp.load + active true + end + task fpcamera.exp.run + active true + end + task fpcamera.revert + active true + end +end + +macro fpcamera.off + task fpcamera.exp.load + active false + end + task fpcamera.exp.run + active false + end + task fpcamera.revert + active false + end +end + +macro fpcamera.revert.on + task fpcamera.revert + active true + end +end + +macro fpcamera.revert.off + task fpcamera.revert + active false + end +end + +# this variable will cycle through the known database names +$fpcamera_DB = 0 +$fpcamera_revert_DB = 0 + +# select images ready for fpcamera analysis +# new entries are added to fpcamPendingExp +# skip already-present entries +task fpcamera.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/fpcamera.exp.log + + task.exec + if ($LABEL:n == 0) break + $run = fpcamtool -pendingexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fpcamera_DB + $run = $run -dbname $DB:$fpcamera_DB + $fpcamera_DB ++ + if ($fpcamera_DB >= $DB:n) set fpcamera_DB = 0 + end + $run = $run -limit $POLL_LIMIT_FPCAMERA + # NOTE: we do not want to overload the dvo db machine with getstar queries, + # so we will limit the fpcamera stage to a smaller number than the other stages + # add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fpcamPendingExp -key fpcam_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fpcamPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fpcamPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the fpcamera_exp script on pending images +task fpcamera.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10 + + task.exec + book npages fpcamPendingExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new images in fpcamPendingExp (pantaskState == INIT) + book getpage fpcamPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fpcamPendingExp $pageName pantaskState RUN + book getword fpcamPendingExp $pageName camera -var CAMERA + book getword fpcamPendingExp $pageName exp_tag -var EXP_TAG + book getword fpcamPendingExp $pageName fpcam_id -var FPCAM_ID + book getword fpcamPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword fpcamPendingExp $pageName path_base -var PATH_BASE + book getword fpcamPendingExp $pageName dbname -var DBNAME + book getword fpcamPendingExp $pageName dvodb -var DVODB + book getword fpcamPendingExp $pageName reduction -var REDUCTION + + # specify choice of remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + if ("$PATH_BASE" == "NULL") + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.fp.%s" $WORKDIR $EXP_TAG $EXP_TAG $FPCAM_ID + else + $outroot = $PATH_BASE + end + + stdout $LOGDIR/fpcamera.exp.log + stderr $LOGDIR/fpcamera.exp.log + + $run = fpcamera_exp.pl --exp_tag $EXP_TAG --fpcam_id $FPCAM_ID --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$DVODB" != "NULL") + $run = $run --dvodb $DVODB + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit fpcamPendingExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword fpcamPendingExp $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword fpcamPendingExp $options:0 pantaskState TIMEOUT + end +end + +task fpcamera.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = fpcamtool -revertprocessedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fpcamera_revert_DB + $run = $run -dbname $DB:$fpcamera_revert_DB + $fpcamera_revert_DB ++ + if ($fpcamera_revert_DB >= $DB:n) set fpcamera_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: trunk/ippTasks/simtest.images.auto =================================================================== --- trunk/ippTasks/simtest.images.auto (revision 42385) +++ trunk/ippTasks/simtest.images.auto (revision 42385) @@ -0,0 +1,22 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR OBJECT-i + check STR "regtool -processedexp -exp_type OBJECT -filter i -inst @CAMERA@ -dbname @DBNAME@" + launch STR "chiptool -dbname @DBNAME@ -definebyquery -inst @CAMERA@ -filter i -set_label simtest -set_data_group simtest_i -set_dist_group simtest -set_reduction PROCESSED -set_workdir @CWD@/work -set_end_stage fake" + block STR NONE +END + +automate METADATA + name STR OBJECT-r + check STR "regtool -processedexp -exp_type OBJECT -filter r -inst @CAMERA@ -dbname @DBNAME@" + launch STR "chiptool -dbname @DBNAME@ -definebyquery -inst @CAMERA@ -filter r -set_label simtest -set_data_group simtest_r -set_dist_group simtest -set_reduction PROCESSED -set_workdir @CWD@/work -set_end_stage fake" + block STR NONE +END Index: trunk/ippTasks/simtest.images.config =================================================================== --- trunk/ippTasks/simtest.images.config (revision 42385) +++ trunk/ippTasks/simtest.images.config (revision 42385) @@ -0,0 +1,77 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 +END + + +## SEQUENCE METADATA +## OBSTYPE STR OBJECT +## CAMERA STR SIMTEST +## +## ### Carina +## CENTER.RA F32 270.75 +## CENTER.DEC F32 -23.7 +## ### COSMOS field +## # CENTER.RA F32 150.119167 +## # CENTER.DEC F32 2.205833 +## +## OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) +## OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) +## OFFSET.NR S32 1 +## OFFSET.ND S32 1 +## +## DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) +## DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) +## DITHER.NR S32 1 +## DITHER.ND S32 1 +## +## DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest +## FILTERS STR r,r,i,i +## @EXPTIMES F32 5.0, 240.0, 5.0, 240.0 +## @SKYMAGS F32 20.86,20.86,20.15,20.15 +## +## # XXX use a more realistic IQ distribution... +## IQ_MIN F32 0.55 +## IQ_MAX F32 1.25 +## +## # rotation sequence +## POS_MIN F32 0.0 +## POS_MAX F32 0.0 +## POS_DELTA F32 1.0 +## END + Index: trunk/ippTasks/simtest.pro =================================================================== --- trunk/ippTasks/simtest.pro (revision 42098) +++ trunk/ippTasks/simtest.pro (revision 42385) @@ -66,7 +66,9 @@ if (("$init" == "new") || ("$init" == "inject")) # XXX this will fail and exit the script if the db does not exist or is old... + # XXX this step is too dangerous and was eliminated exec pxadmin -delete -dbname $dbname # XXX this gives warnings if the db exists... + # XXX this step is too dangerous and was eliminated exec pxadmin -create -dbname $dbname @@ -74,5 +76,8 @@ # the labels "wait" and "proc" are special names used in automate.pro - $ppsim = "ppSimSequence $MODULES:0/$SIMTEST_SEQUENCE simtest.mkimages simtest.inject -path $SIMTEST_RAWDIR -workdir $SIMTEST_WORKDIR -dbname $dbname -label wait -dvodb DVO -tess_id TESS" + # make a reference database + mkref refcat 10 + + $ppsim = "ppSimSequence $SIMTEST_SEQUENCE simtest.mkimages simtest.inject -path $SIMTEST_RAWDIR -workdir $SIMTEST_WORKDIR -dbname $dbname -label wait -dvodb DVO -tess_id TESS -refcat refcat.catdir" if ("$PPSIM_RECIPE" != "default") $ppsim = $ppsim -ppsim_recipe $PPSIM_RECIPE @@ -101,7 +106,11 @@ module automate.pro - module.tasks - - add.label proc + module chip.pro + module camera.pro + module register.pro + # module.tasks + + # XXX consolidate design of label / state / ?? for processing sequences + add.label simtest if ($SIMTEST_THREADS == 0) @@ -115,5 +124,5 @@ automate.load $SIMTEST_AUTO $SIMTEST_CAMERA $dbname - tasks.revert.off + # tasks.revert.off run @@ -159,2 +168,68 @@ $SIMTEST_AUTO = simtest.flatcorr.auto end + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 +$BaseOptions = $BaseOptions -nx 2500 -ny 2500 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 +$RefOptions = $RefOptions -D PSF.MODEL PS_MODEL_GAUSS +$RefOptions = $RefOptions -Df STARS.SIGMA.LIM 0.5 +$RefOptions = $RefOptions -Db PSF.CONVOLVE T + +# create a reference database of fake stars to be used by ppSim below +macro mkref + if ($0 != 3) + echo "mkref (refbase) (density)" + break + end + + local refbase + $refbase = $1 + + exec rm -rf $refbase.catdir + exec rm -f $refbase.fits + + $RefOptions = $RefOptions -Df STARS.DENSITY $2 + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR $refbase.catdir + + exec ppSim $RefOptions $RefConfig $refbase + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR $refbase.catdir -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes $refbase.cmf -quick-airmass + exec relphot -averages -D CATDIR $refbase.catdir -update -region 260 280 -33 -13 +end + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end +