Index: /branches/eam_branches/ipp-pstamp-20260421/ippTasks/Makefile.am =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/Makefile.am (revision 43108) +++ /branches/eam_branches/ipp-pstamp-20260421/ippTasks/Makefile.am (revision 43109) @@ -16,6 +16,4 @@ fake.pro \ warp.pro \ - magic.pro \ - magic.cleanup.pro \ destreak.pro \ destreak.cleanup.pro \ @@ -28,5 +26,4 @@ replicate.pro \ rawcheck.pro \ - diskbalance.pro \ dist.pro \ dist.cleanup.pro \ @@ -56,31 +53,12 @@ fullforce.pro -other_files = \ - ipphosts.manoa.config \ - ipphosts.mhpcc.config \ - site.manoa.pro \ - site.mhpcc.pro \ - surveys.mhpcc.config \ - simtest.pro \ - simtest.basic.config \ - simtest.basic.auto \ - simtest.nebulous.basic.auto \ - simtest.detverify.config \ - simtest.detverify.auto \ - simtest.flatcorr.config \ - simtest.flatcorr.auto \ - simtest.ctemask.config \ - simtest.ctemask.auto \ - simtest.stack.config \ - simtest.stack.auto - pantasksdir = $(datadir)/pantasks ipptasksdir = $(pantasksdir)/modules -ipptasks_DATA = $(task_files) $(other_files) +ipptasks_DATA = $(task_files) install-data-hook: chmod 0755 $(panstasksdir) $(ipptasksdir) -EXTRA_DIST = $(task_files) $(other_files) +EXTRA_DIST = $(task_files) ACLOCAL_AMFLAGS = -I m4 Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/ipphosts.manoa.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/ipphosts.manoa.config (revision 43108) +++ (revision ) @@ -1,19 +1,0 @@ -## basic chiphost table: use the defaults for everything - -chiphosts MULTI - -chiphosts METADATA - camera STR MEGACAM -END - -chiphosts METADATA - camera STR CFH12K -END - -chiphosts METADATA - camera STR GPC1 -END - -chiphosts METADATA - camera STR gpc1 -END Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/ipphosts.mhpcc.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/ipphosts.mhpcc.config (revision 43108) +++ (revision ) @@ -1,410 +1,0 @@ -ipphosts MULTI - -# In this file, we assign all storage hosts to chip and skycells any -# hosts which are having problems are removed as active volumes from -# nebulous. data targeted to those machines are simply re-distributed -# randomly. the only exception is ipp004, which is the DVO db host - -# I've allocated hosts based on disk size. -# 40TB => 2 units of skycells -# 20TB => 1 unit of skycells -# Each host gets one OTA/chip of data. -# These hosts are skipped: -# ipp004 : dvo -# ipp005 : dvo -# ipp026 : crashes -# ipp029 : crashes -# ipp045 : reference cat -## -## PSNSC -- above is way out of date -- update 20170112 in anticipation of -## nodes 2 units of skycells -# 20TB => 1 unit of skycells -# Each host gets one OTA/chip of data. -# These hosts are skipped: -# ipp004 : dvo -# ipp005 : dvo -# ipp026 : crashes -# ipp029 : crashes -# ipp045 : reference cat -## -## now focus on new storage nodes only 12/18/14 -## -- focus on lower used nodes as well, limit 067-081 since warp,stack,diff permament -## -- MD also typically x1--x8, 10--80 -- so need to try and balance that.. -## -- easier if raise the count to 90, then 00-10,90-100 map the same for MD as low -ipphosts METADATA - camera STR skycell - count S32 90 - sky00 STR ipp067.0 - sky01 STR ipp068.0 - sky02 STR ipp069.0 - sky03 STR ipp070.0 - sky04 STR ipp071.0 - sky05 STR ipp072.0 - sky06 STR ipp073.0 - sky07 STR ipp074.0 - sky08 STR ipp075.0 - sky09 STR ipp076.0 - sky10 STR ipp077.0 - sky11 STR ipp078.0 - - sky12 STR ipp077.0 - sky13 STR ipp071.0 - sky14 STR ipp072.0 - sky15 STR ipp073.0 - sky16 STR ipp074.0 - sky17 STR ipp075.0 - - sky18 STR ipp079.0 - sky19 STR ipp080.0 - sky20 STR ipp081.0 - - sky21 STR ipp089.0 - sky22 STR ipp088.0 - sky23 STR ipp087.0 - sky24 STR ipp086.0 - sky25 STR ipp085.0 - sky26 STR ipp084.0 - sky27 STR ipp083.0 - sky28 STR ipp082.0 - - sky29 STR ipp067.0 - sky30 STR ipp068.0 - - sky31 STR ipp095.0 - sky32 STR ipp095.0 - sky33 STR ipp093.0 - sky34 STR ipp093.0 - sky35 STR ipp090.0 - sky36 STR ipp090.0 - sky37 STR ipp089.0 - sky38 STR ipp088.0 - - sky39 STR ipp069.0 - sky40 STR ipp070.0 - - sky41 STR ipp087.0 - sky42 STR ipp086.0 - sky43 STR ipp085.0 - sky44 STR ipp084.0 - sky45 STR ipp083.0 - sky46 STR ipp082.0 - sky47 STR ipp095.0 - sky48 STR ipp095.0 - - sky49 STR ipp071.0 - sky50 STR ipp072.0 - - sky51 STR ipp093.0 - sky52 STR ipp093.0 - sky53 STR ipp090.0 - sky54 STR ipp090.0 - sky55 STR ipp089.0 - sky56 STR ipp088.0 - sky57 STR ipp087.0 - sky58 STR ipp086.0 - - sky59 STR ipp073.0 - sky60 STR ipp074.0 - - sky61 STR ipp085.0 - sky62 STR ipp084.0 - sky63 STR ipp083.0 - sky64 STR ipp082.0 - sky65 STR ipp095.0 - sky66 STR ipp095.0 - sky67 STR ipp093.0 - sky68 STR ipp093.0 - - sky69 STR ipp075.0 - sky70 STR ipp076.0 - - sky71 STR ipp090.0 - sky72 STR ipp090.0 - sky73 STR ipp089.0 - sky74 STR ipp088.0 - sky75 STR ipp087.0 - sky76 STR ipp086.0 - sky77 STR ipp085.0 - sky78 STR ipp084.0 - - sky79 STR ipp077.0 - sky80 STR ipp078.0 - sky81 STR ipp079.0 - - sky82 STR ipp076.0 - sky83 STR ipp077.0 - sky84 STR ipp078.0 - sky85 STR ipp079.0 - sky86 STR ipp080.0 - sky87 STR ipp081.0 - - sky88 STR ipp080.0 - sky89 STR ipp081.0 -END - -# These are ordered oddly to prevent excessive targetting shuffles. -## now focus on new storage nodes only 12/18/14 -## -- no limit loading on lower 067-081 since chips transient -ipphosts METADATA - camera STR GPC1 - XY01 STR ipp095 - XY02 STR ipp095 - XY03 STR ipp093 - XY04 STR ipp093 - XY05 STR ipp090 - XY06 STR ipp090 - XY10 STR ipp089 - XY11 STR ipp088 - XY12 STR ipp087 - XY13 STR ipp086 - XY14 STR ipp085 - XY15 STR ipp084 - XY16 STR ipp083 - XY17 STR ipp082 - XY20 STR ipp081 - XY21 STR ipp080 - XY22 STR ipp079 - XY23 STR ipp078 - XY24 STR ipp077 - XY25 STR ipp076 - XY26 STR ipp075 - XY27 STR ipp074 - XY30 STR ipp073 - XY31 STR ipp072 - XY32 STR ipp071 - XY33 STR ipp070 - XY34 STR ipp069 - XY35 STR ipp068 - XY36 STR ipp067 - - XY37 STR ipp095 - XY40 STR ipp095 - XY41 STR ipp093 - XY42 STR ipp093 - XY43 STR ipp090 - XY44 STR ipp090 - XY45 STR ipp089 - XY46 STR ipp088 - XY47 STR ipp087 - XY50 STR ipp086 - XY51 STR ipp085 - XY52 STR ipp084 - XY53 STR ipp083 - XY54 STR ipp082 - XY55 STR ipp081 - XY56 STR ipp080 - XY57 STR ipp079 - XY60 STR ipp078 - XY61 STR ipp077 - XY62 STR ipp076 - XY63 STR ipp075 - XY64 STR ipp074 - XY65 STR ipp073 - XY66 STR ipp072 - XY67 STR ipp071 - XY71 STR ipp070 - XY72 STR ipp069 - XY73 STR ipp068 - XY74 STR ipp067 - - XY75 STR ipp087 - XY76 STR ipp084 -END - -## focus nightly processing ota on disks with more space 12/18/14 -## -- ipp066 and lower just raw ota products until full -## -- shouldn't be used for reprocessing? -ipphosts METADATA - camera STR gpc1 - ota01 STR ipp067 - ota02 STR ipp068 - ota03 STR ipp069 - ota04 STR ipp070 - ota05 STR ipp071 - ota06 STR ipp072 - ota10 STR ipp073 - ota11 STR ipp074 - ota12 STR ipp075 - ota13 STR ipp076 - ota14 STR ipp077 - ota15 STR ipp078 - ota16 STR ipp079 - ota17 STR ipp080 - ota20 STR ipp081 - ota21 STR ipp082 - ota22 STR ipp083 - ota23 STR ipp084 - ota24 STR ipp085 - ota25 STR ipp086 - ota26 STR ipp087 - ota27 STR ipp088 - ota30 STR ipp089 - ota31 STR ipp090 - ota32 STR ipp090 - ota33 STR ipp093 - ota34 STR ipp093 - ota35 STR ipp095 - ota36 STR ipp095 - - ota37 STR ipp067 - ota40 STR ipp068 - ota41 STR ipp069 - ota42 STR ipp070 - ota43 STR ipp071 - ota44 STR ipp072 - ota45 STR ipp073 - ota46 STR ipp074 - ota47 STR ipp075 - ota50 STR ipp076 - ota51 STR ipp077 - ota52 STR ipp078 - ota53 STR ipp079 - ota54 STR ipp080 - ota55 STR ipp081 - ota56 STR ipp082 - ota57 STR ipp083 - ota60 STR ipp084 - ota61 STR ipp085 - ota62 STR ipp086 - ota63 STR ipp087 - ota64 STR ipp088 - ota65 STR ipp089 - ota66 STR ipp090 - ota67 STR ipp090 - ota71 STR ipp093 - ota72 STR ipp093 - ota73 STR ipp095 - ota74 STR ipp095 - - ota75 STR ipp060 - ota76 STR ipp060 -END - -ipphosts METADATA - camera STR distribution - count S32 26 - 0 STR ipp021 - 1 STR ipp023 - 2 STR ipp024 - 3 STR ipp026 - 4 STR ipp028 - 5 STR ipp029 - 6 STR ipp030 - 7 STR ipp031 - 8 STR ipp032 - 9 STR ipp033 - 10 STR ipp034 - 11 STR ipp035 - 12 STR ipp036 - 13 STR ipp038 - 14 STR ipp039 - 15 STR ipp040 - 16 STR ipp041 - 17 STR ipp043 - 18 STR ipp044 - 19 STR ipp045 - 20 STR ipp046 - 21 STR ipp047 - 22 STR ipp048 - 23 STR ipp050 - 24 STR ipp051 - 25 STR ipp052 -END - -ipphosts METADATA - camera STR dist_chip - count S32 60 - - XY01 STR ipp010 - XY02 STR ipp011 - XY03 STR ipp006 - XY04 STR ipp054 - - XY05 STR ipp007 - XY06 STR ipp055 - XY10 STR ipp008 - XY11 STR ipp056 - - XY12 STR ipp009 - XY13 STR ipp057 - XY14 STR ipp058 - XY15 STR ipp059 - - XY16 STR ipp012 - XY17 STR ipp060 - XY20 STR ipp013 - XY21 STR ipp014 - - XY22 STR ipp061 - XY23 STR ipp015 - XY24 STR ipp016 - XY25 STR ipp062 - - XY26 STR ipp064 - XY27 STR ipp065 - XY30 STR ipp066 - XY31 STR ipp020 - - XY32 STR ipp063 - XY33 STR ipp021 - XY34 STR ipp023 - XY35 STR ipp013 - - XY36 STR ipp024 - XY37 STR ipp019 - XY40 STR ipp025 - XY41 STR ipp018 - - XY42 STR ipp017 - XY43 STR ipp015 - XY44 STR ipp027 - XY45 STR ipp028 - - XY46 STR ipp029 - XY47 STR ipp030 - XY50 STR ipp031 - XY51 STR ipp032 - - XY52 STR ipp033 - XY53 STR ipp034 - XY54 STR ipp035 - XY55 STR ipp036 - - XY56 STR ipp037 - XY57 STR ipp038 - XY60 STR ipp039 - XY61 STR ipp040 - - XY62 STR ipp041 - XY63 STR ipp042 - XY64 STR ipp043 - XY65 STR ipp044 - - XY66 STR ipp046 - XY67 STR ipp047 - XY71 STR ipp048 - XY72 STR ipp049 - - XY73 STR ipp050 - XY74 STR ipp051 - XY75 STR ipp052 - XY76 STR ipp053 -END - -ipphosts METADATA - camera STR dist_skycell - count S32 86 - sky00 STR ipp006 - sky01 STR ipp006 - sky02 STR ipp007 - sky03 STR ipp007 - sky04 STR ipp008 - sky05 STR ipp008 - sky06 STR ipp009 - sky07 STR ipp009 - sky08 STR ipp010 - sky09 STR ipp010 - sky10 STR ipp011 - sky11 STR ipp011 - sky12 STR ipp012 - sky13 STR ipp012 - sky14 STR ipp013 - sky15 STR ipp013 - sky16 STR ipp014 - sky17 STR ipp014 - sky18 STR ipp015 - sky19 STR ipp015 - sky20 STR ipp016 - sky21 STR ipp016 - sky22 STR ipp017 - sky23 STR ipp017 - sky24 STR ipp018 - sky25 STR ipp018 - sky26 STR ipp019 - sky27 STR ipp019 - sky28 STR ipp020 - sky29 STR ipp020 - sky30 STR ipp021 - sky31 STR ipp021 - sky32 STR ipp023 - sky33 STR ipp024 - sky34 STR ipp025 - sky35 STR ipp027 - sky36 STR ipp028 - sky37 STR ipp030 - sky38 STR ipp031 - sky39 STR ipp032 - sky40 STR ipp033 - sky41 STR ipp034 - sky42 STR ipp035 - sky43 STR ipp036 - sky44 STR ipp037 - sky45 STR ipp038 - sky46 STR ipp039 - sky47 STR ipp040 - sky48 STR ipp041 - sky49 STR ipp042 - sky50 STR ipp043 - sky51 STR ipp044 - sky52 STR ipp046 - sky53 STR ipp047 - sky54 STR ipp048 - sky55 STR ipp049 - sky56 STR ipp050 - sky57 STR ipp051 - sky58 STR ipp052 - sky59 STR ipp053 - sky60 STR ipp054 - sky61 STR ipp054 - sky62 STR ipp055 - sky63 STR ipp055 - sky64 STR ipp056 - sky65 STR ipp056 - sky66 STR ipp057 - sky67 STR ipp057 - sky68 STR ipp058 - sky69 STR ipp058 - sky70 STR ipp059 - sky71 STR ipp059 - sky72 STR ipp060 - sky73 STR ipp060 - sky74 STR ipp061 - sky75 STR ipp061 - sky76 STR ipp062 - sky77 STR ipp062 - sky78 STR ipp063 - sky79 STR ipp063 - sky80 STR ipp064 - sky81 STR ipp064 - sky82 STR ipp065 - sky83 STR ipp065 - sky84 STR ipp066 - sky85 STR ipp066 -END - -ipphosts METADATA - camera STR STARE - - ota01 STR stare00 - ota02 STR stare01 - ota03 STR stare02 - ota04 STR stare04 - - ota05 STR stare00 - ota06 STR stare01 - ota10 STR stare02 - ota11 STR stare04 - - ota12 STR stare00 - ota13 STR stare01 - ota14 STR stare02 - ota15 STR stare04 - - ota16 STR stare00 - ota17 STR stare01 - ota20 STR stare02 - ota21 STR stare04 - - ota22 STR stare00 - ota23 STR stare01 - ota24 STR stare02 - ota25 STR stare04 - - ota26 STR stare00 - ota27 STR stare01 - ota30 STR stare02 - ota31 STR stare04 - - ota32 STR stare00 - ota33 STR stare01 - ota34 STR stare02 - ota35 STR stare04 - - ota36 STR stare00 - ota37 STR stare01 - ota40 STR stare02 - ota41 STR stare04 - - ota42 STR stare00 - ota43 STR stare01 - ota44 STR stare02 - ota45 STR stare04 - - ota46 STR stare00 - ota47 STR stare01 - ota50 STR stare02 - ota51 STR stare04 - - ota52 STR stare00 - ota53 STR stare01 - ota54 STR stare02 - ota55 STR stare04 - - ota56 STR stare00 - ota57 STR stare01 - ota60 STR stare02 - ota61 STR stare04 - - ota62 STR stare00 - ota63 STR stare01 - ota64 STR stare02 - ota65 STR stare04 - - ota66 STR stare00 - ota67 STR stare01 - ota71 STR stare02 - ota72 STR stare04 - - ota73 STR stare00 - ota74 STR stare01 - ota75 STR stare02 - ota76 STR stare04 -END - - -# Backup of old stare allocation setup. -# ipphosts METADATA -# camera STR STARE - -# ota01 STR ipp010 -# ota02 STR ipp010 -# ota03 STR ipp010 -# ota04 STR ipp010 - -# ota05 STR ipp011 -# ota06 STR ipp011 -# ota10 STR ipp011 -# ota11 STR ipp011 - -# ota12 STR ipp012 -# ota13 STR ipp012 -# ota14 STR ipp012 -# ota15 STR ipp012 - -# ota16 STR ipp013 -# ota17 STR ipp013 -# ota20 STR ipp013 -# ota21 STR ipp013 - -# ota22 STR ipp014 -# ota23 STR ipp014 -# ota24 STR ipp014 -# ota25 STR ipp014 - -# ota26 STR ipp015 -# ota27 STR ipp015 -# ota30 STR ipp015 -# ota31 STR ipp015 - -# ota32 STR ipp016 -# ota33 STR ipp016 -# ota34 STR ipp016 -# ota35 STR ipp016 - -# ota36 STR ipp017 -# ota37 STR ipp017 -# ota40 STR ipp017 -# ota41 STR ipp017 - -# ota42 STR ipp018 -# ota43 STR ipp018 -# ota44 STR ipp018 -# ota45 STR ipp018 - -# ota46 STR ipp019 -# ota47 STR ipp019 -# ota50 STR ipp019 -# ota51 STR ipp019 - -# ota52 STR ipp020 -# ota53 STR ipp020 -# ota54 STR ipp020 -# ota55 STR ipp020 - -# ota56 STR ipp021 -# ota57 STR ipp021 -# ota60 STR ipp021 -# ota61 STR ipp021 - -# ota62 STR ipp010 -# ota63 STR ipp011 -# ota64 STR ipp012 -# ota65 STR ipp013 - -# ota66 STR ipp014 -# ota67 STR ipp015 -# ota71 STR ipp016 -# ota72 STR ipp017 - -# ota73 STR ipp018 -# ota74 STR ipp019 -# ota75 STR ipp020 -# ota76 STR ipp021 -# END Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.basic.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.basic.auto (revision 43108) +++ (revision ) @@ -1,128 +1,0 @@ - -automate MULTI - -# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK -# success on CHECK -> LAUNCH -# failure on BLOCK -> CHECK - -# a stage starts in state INIT.BLOCK. - -# the "automate.block" task runs the 'block' command regularly. if it -# returns a valid result, the block is set and the state remains in -# block. if there is no result (ie, no matching BIAS, etc), the stage -# is put to state INIT.CHECK. - -# the "automate.check" tasks runs the 'check' command regularly. if -# it returns ncheck valid entries (default ncheck = 1), the state goes -# to INIT.LAUNCH. - -# the "automate.launch" tasks runs the 'launch' command once. if it returns -# ncheck valid entries (default ncheck = 1), the state goes to -# INIT.LAUNCH. - -# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis - -automate METADATA - name STR BIAS - check STR "regtool -dbname @DBNAME@ -processedexp -exp_type BIAS -inst @CAMERA@" - ncheck S32 20 - launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS" - block STR "dettool -dbname @DBNAME@ -runs -active -det_type BIAS" -END - -automate METADATA - name STR DARK - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type BIAS -time 2010/01/01" - launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK" - block STR "dettool -dbname @DBNAME@ -runs -active -det_type DARK" -END - -automate METADATA - name STR SHUTTER - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type DARK -time 2010/01/01" - launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r" - block STR "dettool -dbname @DBNAME@ -runs -active -det_type SHUTTER" -END - -automate METADATA - name STR FLAT-r - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type SHUTTER -time 2010/01/01" - launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r" - block STR "dettool -dbname @DBNAME@ -runs -active -det_type FLAT -filter r" -END - -automate METADATA - name STR FLAT-i - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type SHUTTER -time 2010/01/01" - launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i" - block STR "dettool -dbname @DBNAME@ -runs -active -det_type FLAT -filter i" -END - -## the lines below assume the simtest suite injects the chips with an end state of chip, but currently they only go to 'reg' -automate METADATA - name STR OBJECT-i - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -filter i -time 2010/01/01" -# launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -label wait -filter i -set_label proc -set_data_group simtest_i -set_dist_group simtest" - launch STR "chiptool -dbname @DBNAME@ -definebyquery -inst @CAMERA@ -filter i -set_label proc -set_data_group simtest_i -set_dist_group simtest -set_workdir file://@CWD@/chip" - block STR "chiptool -dbname @DBNAME@ -listrun -filter i -label proc" -END - -automate METADATA - name STR OBJECT-r - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -filter r -time 2010/01/01" -# launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -label wait -filter r -set_label proc -set_data_group simtest_r -set_dist_group simtest" - launch STR "chiptool -dbname @DBNAME@ -definebyquery -inst @CAMERA@ -filter r -set_label proc -set_data_group simtest_i -set_dist_group simtest -set_workdir file://@CWD@/chip" - block STR "chiptool -dbname @DBNAME@ -listrun -filter r -label proc" -END - -### ### Stack automation -### automate METADATA -### name STR STACK -### regular STR "stacktool -dbname @DBNAME@ -definebyquery -set_label proc -set_data_group simtest_r -set_dist_group simtest -set_workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -min_num 4" -### END -### -### ### Diff automation -### automate METADATA -### name STR DIFF -### regular STR "difftool -dbname @DBNAME@ -definewarpstack -set_label proc -set_workdir file://@CWD@/diff -good_frac 0.2" -### END - -### automate METADATA -### name STR OBJECT-r -### check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -filter r" -### launch STR "chiptool -dbname @DBNAME@ -unblock -label simtest-r" -### block STR "chiptool -dbname @DBNAME@ -unmasked -label simtest-r" -### END - -### automate METADATA -### name STR OBJECT-i -### check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -filter i" -### launch STR "chiptool -dbname @DBNAME@ -unblock -label simtest-i" -### block STR "chiptool -dbname @DBNAME@ -unmasked -label simtest-i" -### END - -### there is a weakness in the label / block business: the labels are -### generic, and the blocks are against those fairly generic words. -### that makes it difficult to block and unblock science exposures -### based on different detrend types that are available. For example, -### to block the r-band against the absence of the desired r-band -### flat, we would need to specify a label specific to the r-band -### images and remove that block when ready. - -### a better approach might be to modify the labels rather than the -### blocks. we can define, eg, chiptool -set_label -definebyquery to -### turn assign the label names based on various properties of the -### interesting images. - -### here is what the automate element might look like for such a circumstance: - -### automate METADATA -### name STR OBJECT-i -### block STR "chiptool -dbname @DBNAME@ -unmasked -label wait -filter i -time_begin 2008/1/1 -time_end 2008/1/2" -### check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -filter i -time_begin 2008/1/1 -time_end 2008/1/2" -### launch STR "chiptool -dbname @DBNAME@ -set_label proc -label wait -time_begin 2008/1/1 -time_end 2008/1/2" -### END - -### XXX still not quite there.... -### the 'checks' or maybe the 'launch' needs to include a '-n_exp_min' ? -### Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.basic.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.basic.config (revision 43108) +++ (revision ) @@ -1,115 +1,0 @@ - -# allows for multi-file outputs to be selected by ppSimSequence -FILERULE STR *fits - -# define the simulated data to be generated -SEQUENCE MULTI - -SEQUENCE METADATA - OBSTYPE STR BIAS - CAMERA STR SIMTEST - NIMAGES S32 20 -END - -SEQUENCE METADATA - OBSTYPE STR DARK - CAMERA STR SIMTEST - @EXPTIMES F32 30.0, 300.0 - @NIMAGES S32 10, 10 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR SIMTEST - - FILTERS STR r,r,i - @EXPTIMES F32 0.1,20.0,20.0 - - NSETUP S32 5 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR SIMTEST - - FILTERS STR r,r,r,r,r - @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 - - NSETUP S32 1 -END - -SEQUENCE METADATA - OBSTYPE STR OBJECT - CAMERA STR SIMTEST - -### Carina - CENTER.RA F32 270.75 - CENTER.DEC F32 -23.7 -### COSMOS field -# CENTER.RA F32 150.119167 -# CENTER.DEC F32 2.205833 - - OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 1 - OFFSET.ND S32 1 - - DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 3 - DITHER.ND S32 2 - - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest - FILTERS STR r,i - @EXPTIMES F32 10.00,10.00 - @SKYMAGS F32 20.86,20.15 - - # XXX use a more realistic IQ distribution... - IQ_MIN F32 0.55 - IQ_MAX F32 1.25 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 - -END - - -SEQUENCE METADATA - OBSTYPE STR OBJECT - CAMERA STR SIMTEST - -### Carina - CENTER.RA F32 270.75 - CENTER.DEC F32 -23.7 -### COSMOS field -# CENTER.RA F32 150.119167 -# CENTER.DEC F32 2.205833 - - OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 1 - OFFSET.ND S32 1 - - DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 1 - DITHER.ND S32 1 - - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest - FILTERS STR r,r,i,i - @EXPTIMES F32 5.0, 240.0, 5.0, 240.0 - @SKYMAGS F32 20.86,20.86,20.15,20.15 - - # XXX use a more realistic IQ distribution... - IQ_MIN F32 0.55 - IQ_MAX F32 1.25 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 - -END - Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.ctemask.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.ctemask.auto (revision 43108) +++ (revision ) @@ -1,39 +1,0 @@ - -automate MULTI - -# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK -# success on CHECK -> LAUNCH -# failure on BLOCK -> CHECK - -# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis - -automate METADATA - name STR BIAS - check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" - ncheck S32 5 - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" - block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" -END - -automate METADATA - name STR DARK - check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" - block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" -END - -automate METADATA - name STR SHUTTER - check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" - block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" -END - -automate METADATA - name STR FLAT-r - check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" - block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" -END - -# XXX let's build the necessary data, then see how to run the cte mask generation (input is flattened flats) Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.ctemask.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.ctemask.config (revision 43108) +++ (revision ) @@ -1,38 +1,0 @@ - -# very basic test to generate flats and test the CTE masking process -# define the simulated data to be generated -SEQUENCE MULTI - -SEQUENCE METADATA - OBSTYPE STR BIAS - CAMERA STR SIMTEST - NIMAGES S32 5 -END - -SEQUENCE METADATA - OBSTYPE STR DARK - CAMERA STR SIMTEST - @EXPTIMES F32 30.0, 300.0 - @NIMAGES S32 3, 3 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR SIMTEST - - FILTERS STR r,r,r,r,r - @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 - - NSETUP S32 1 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR SIMTEST - - FILTERS STR r - @EXPTIMES F32 20.0 - - NSETUP S32 7 -END - Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.det =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.det (revision 43108) +++ (revision ) @@ -1,42 +1,0 @@ - -detRunDef MULTI - -detRunDef METADATA - detRunName STR BIAS - prereq STR NONE - options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type BIAS -select_exp_type BIAS" - dbname STR eamtest - pantaskState STR NULL -END - -detRunDef METADATA - detRunName STR DARK - prereq STR BIAS - options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK" - dbname STR eamtest - pantaskState STR NULL -END - -detRunDef METADATA - detRunName STR SHUTTER - prereq STR DARK - options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r" - dbname STR eamtest - pantaskState STR NULL -END - -detRunDef METADATA - detRunName STR FLAT-r - prereq STR SHUTTER - options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r" - dbname STR eamtest - pantaskState STR NULL -END - -detRunDef METADATA - detRunName STR FLAT-i - prereq STR SHUTTER - options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i" - dbname STR eamtest - pantaskState STR NULL -END Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.detverify.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.detverify.auto (revision 43108) +++ (revision ) @@ -1,78 +1,0 @@ - -automate MULTI - -# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK -# success on CHECK -> LAUNCH -# failure on BLOCK -> CHECK - -automate METADATA - name STR BIAS - check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" - ncheck S32 20 - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" - block STR "dettool -runs -active -det_type BIAS -mode master -dbname @DBNAME@" -END - -automate METADATA - name STR BIAS_VERIFY - check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" - block STR "dettool -runs -active -det_type BIAS -mode verify -dbname @DBNAME@" -END - -automate METADATA - name STR DARK - check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" - block STR "dettool -runs -active -det_type DARK -mode master -dbname @DBNAME@" -END - -automate METADATA - name STR DARK_VERIFY - check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" - block STR "dettool -runs -active -det_type DARK -mode verify -dbname @DBNAME@" -END - -automate METADATA - name STR SHUTTER - check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" - block STR "dettool -runs -active -det_type SHUTTER -mode master -dbname @DBNAME@" -END - -automate METADATA - name STR SHUTTER_VERIFY - check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" - block STR "dettool -runs -active -det_type FLAT -filter r -mode verify -dbname @DBNAME@" -END - -automate METADATA - name STR FLAT-r - check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" - block STR "dettool -runs -active -det_type FLAT -filter r -mode master -dbname @DBNAME@" -END - -automate METADATA - name STR FLAT-i - check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -dbname @DBNAME@" - block STR "dettool -runs -active -det_type FLAT -filter i -mode master -dbname @DBNAME@" -END - -automate METADATA - name STR FLAT_VERIFY-r - check STR "detselect -search -inst @CAMERA@ -det_type FLAT -filter r -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" - block STR "dettool -runs -active -det_type FLAT -filter r -mode verify -dbname @DBNAME@" -END - -automate METADATA - name STR FLAT_VERIFY-i - check STR "detselect -search -inst @CAMERA@ -det_type FLAT -filter i -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" - block STR "dettool -runs -active -det_type FLAT -filter i -mode verify -dbname @DBNAME@" -END - Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.detverify.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.detverify.config (revision 43108) +++ (revision ) @@ -1,37 +1,0 @@ - -# define the simulated data to be generated -SEQUENCE MULTI - -SEQUENCE METADATA - OBSTYPE STR BIAS - CAMERA STR SIMTEST - NIMAGES S32 20 -END - -SEQUENCE METADATA - OBSTYPE STR DARK - CAMERA STR SIMTEST - @EXPTIMES F32 30.0, 300.0 - @NIMAGES S32 10, 10 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR SIMTEST - - FILTERS STR r,r,i - @EXPTIMES F32 0.1,20.0,20.0 - - NSETUP S32 5 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR SIMTEST - - FILTERS STR r,r,r,r,r - @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 - - NSETUP S32 1 -END - Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.flatcorr.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.flatcorr.auto (revision 43108) +++ (revision ) @@ -1,76 +1,0 @@ - -automate MULTI - -# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK -# success on CHECK -> LAUNCH -# failure on BLOCK -> CHECK - -# this is a simplified automate script which only models BIAS and FLAT. -# generate appropriate simulated data with the FLATCORR recipe - -# wait until 20 bias frames are registered, then define a new bias run -automate METADATA - name STR BIAS - check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" - ncheck S32 10 - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" - block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" -END - -# wait until the master bias frame is registered, then define a new dark run -automate METADATA - name STR DARK - check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" - block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" -END - -# wait until the master dark frame is registered, then define a new shutter run -automate METADATA - name STR SHUTTER - check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" - block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" -END - -# wait until the master bias frame is registered, then define a new flat-r run -automate METADATA - name STR RAWFLAT-r - check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT_RAW -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" - block STR "dettool -runs -active -det_type FLAT_RAW -filter r -dbname @DBNAME@" -END - -# wait until the master flat-r frame is registered, then initiate a flat-field correction analysis -automate METADATA - name STR FLATCORR-r - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT_RAW" - launch STR "flatcorr -definebyquery -dbname @DBNAME@ -det_type FLATCORR -inst @CAMERA@ -exp_type OBJECT -filter r -set_workdir file://@CWD@/detwork -set_reduction FLATCORR -set_dvodb @CWD@/catdir.flatcorr -set_region 270.6,270.9:-23.8,-23.5 -set_filter r" - block STR "dettool -runs -active -det_type FLATCORR -filter r -dbname @DBNAME@" -END - -# wait until the flat-r correction frame is registered, then define the flat-fields to be corrected -# XXX for the moment, the det_id here is hard-wired -automate METADATA - name STR FLATFIX-r - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLATCORR" - launch STR "dettool -makecorrection -dbname @DBNAME@ -det_id 4 -iteration 0 -det_type FLAT -workdir file://@CWD@/detwork" - block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" -END - -# wait until the corrected master flat-r frame is registered, then initiate a second flat-field correction analysis -automate METADATA - name STR FLATTEST-r - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" - launch STR "flatcorr -definebyquery -dbname @DBNAME@ -det_type FLATTEST -inst @CAMERA@ -exp_type OBJECT -filter r -set_workdir file://@CWD@/detwork -set_reduction FLATTEST -set_dvodb @CWD@/catdir.flattest -set_region 270.6,270.9:-23.8,-23.5 -set_filter r" - block STR "dettool -runs -active -det_type FLATCORR -filter r -dbname @DBNAME@" -END - -## # wait until the master flat-r frame is registered, then unblock the r-band data for chip-level processing -## automate METADATA -## name STR OBJECT-i -## check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" -## launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" -## block STR NONE -## END -## Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.flatcorr.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.flatcorr.config (revision 43108) +++ (revision ) @@ -1,189 +1,0 @@ - -# generate a simple fake data set with 2x2 dithers at offset of -# 50, 100, 200 arcsec (camera is 530 arcsec on a side) - -# define the simulated data to be generated -SEQUENCE MULTI - -SEQUENCE METADATA - OBSTYPE STR BIAS - CAMERA STR SIMTEST - NIMAGES S32 10 -END - -SEQUENCE METADATA - OBSTYPE STR DARK - CAMERA STR SIMTEST - @EXPTIMES F32 30.0, 300.0 - @NIMAGES S32 5, 5 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR SIMTEST - - FILTERS STR r,r - @EXPTIMES F32 0.1,20.0 - - NSETUP S32 3 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR SIMTEST - - FILTERS STR r,r,r,r,r - @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 - - NSETUP S32 1 -END - -# a single image at the field center -SEQUENCE METADATA - OBSTYPE STR OBJECT - CAMERA STR SIMTEST - - ### Carina (modestly high density) - CENTER.RA F32 270.75 - CENTER.DEC F32 -23.7 - - # the simtest camera is 2k x 2k, 530 arcsec on a side - # XXX note that we cannot define the offsets in terms of the camera dimensions... - - OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 1 - OFFSET.ND S32 1 - - DITHER.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 1 - DITHER.ND S32 1 - - # XXX is DVODB used? - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest - FILTERS STR r - @EXPTIMES F32 10.00 - @SKYMAGS F32 20.86 - - # modest IQ range here - IQ_MIN F32 0.70 - IQ_MAX F32 0.90 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 - -END - -# dither set @ scale 1 = 60 arcsec -SEQUENCE METADATA - OBSTYPE STR OBJECT - CAMERA STR SIMTEST - - ### Carina (modestly high density) - CENTER.RA F32 270.75 - CENTER.DEC F32 -23.7 - - # XXX note that we cannot define the offsets in terms of the camera dimensions... - OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 1 - OFFSET.ND S32 1 - - DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 2 - DITHER.ND S32 2 - - # XXX is DVODB used? - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest - FILTERS STR r - @EXPTIMES F32 10.00 - @SKYMAGS F32 20.86 - - # modest IQ range here - IQ_MIN F32 0.70 - IQ_MAX F32 0.90 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 - -END - -# dither set @ scale 2 = 120 arcsec -SEQUENCE METADATA - OBSTYPE STR OBJECT - CAMERA STR SIMTEST - - ### Carina (modestly high density) - CENTER.RA F32 270.75 - CENTER.DEC F32 -23.7 - - # XXX note that we cannot define the offsets in terms of the camera dimensions... - OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 1 - OFFSET.ND S32 1 - - DITHER.RA F32 120.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 120.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 2 - DITHER.ND S32 2 - - # XXX is DVODB used? - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest - FILTERS STR r - @EXPTIMES F32 10.00 - @SKYMAGS F32 20.86 - - # modest IQ range here - IQ_MIN F32 0.70 - IQ_MAX F32 0.90 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 - -END - -# dither set @ scale 3 = 240 arcsec -SEQUENCE METADATA - OBSTYPE STR OBJECT - CAMERA STR SIMTEST - - ### Carina (modestly high density) - CENTER.RA F32 270.75 - CENTER.DEC F32 -23.7 - - # XXX note that we cannot define the offsets in terms of the camera dimensions... - OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 1 - OFFSET.ND S32 1 - - DITHER.RA F32 240.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 240.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 2 - DITHER.ND S32 2 - - # XXX is DVODB used? - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest - FILTERS STR r - @EXPTIMES F32 10.00 - @SKYMAGS F32 20.86 - - # modest IQ range here - IQ_MIN F32 0.70 - IQ_MAX F32 0.90 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 - -END Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.images.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.images.auto (revision 43108) +++ (revision ) @@ -1,22 +1,0 @@ - -automate MULTI - -# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK -# success on CHECK -> LAUNCH -# failure on BLOCK -> CHECK - -# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis - -automate METADATA - name STR OBJECT-i - check STR "regtool -processedexp -exp_type OBJECT -filter i -inst @CAMERA@ -dbname @DBNAME@" - launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc -set_data_group simtest_i -set_dist_group simtest" - block STR NONE -END - -automate METADATA - name STR OBJECT-r - check STR "regtool -processedexp -exp_type OBJECT -filter r -inst @CAMERA@ -dbname @DBNAME@" - launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc -set_data_group simtest_r -set_dist_group simtest" - block STR NONE -END Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.images.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.images.config (revision 43108) +++ (revision ) @@ -1,77 +1,0 @@ - -# define the simulated data to be generated -SEQUENCE MULTI - -SEQUENCE METADATA - OBSTYPE STR OBJECT - CAMERA STR SIMTEST - -### Carina - CENTER.RA F32 270.75 - CENTER.DEC F32 -23.7 -### COSMOS field -# CENTER.RA F32 150.119167 -# CENTER.DEC F32 2.205833 - - OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 1 - OFFSET.ND S32 1 - - DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 3 - DITHER.ND S32 2 - - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest - FILTERS STR r,i - @EXPTIMES F32 10.00,10.00 - @SKYMAGS F32 20.86,20.15 - - # XXX use a more realistic IQ distribution... - IQ_MIN F32 0.55 - IQ_MAX F32 1.25 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 -END - - -SEQUENCE METADATA - OBSTYPE STR OBJECT - CAMERA STR SIMTEST - -### Carina - CENTER.RA F32 270.75 - CENTER.DEC F32 -23.7 -### COSMOS field -# CENTER.RA F32 150.119167 -# CENTER.DEC F32 2.205833 - - OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 1 - OFFSET.ND S32 1 - - DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 1 - DITHER.ND S32 1 - - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest - FILTERS STR r,r,i,i - @EXPTIMES F32 5.0, 240.0, 5.0, 240.0 - @SKYMAGS F32 20.86,20.86,20.15,20.15 - - # XXX use a more realistic IQ distribution... - IQ_MIN F32 0.55 - IQ_MAX F32 1.25 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 -END - Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.large.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.large.config (revision 43108) +++ (revision ) @@ -1,73 +1,0 @@ - -# define the simulated data to be generated -SEQUENCE MULTI - -SEQUENCE METADATA - OBSTYPE STR BIAS - CAMERA STR SIMTEST - NIMAGES S32 20 -END - -SEQUENCE METADATA - OBSTYPE STR DARK - CAMERA STR SIMTEST - @EXPTIMES F32 30.0, 300.0 - @NIMAGES S32 10, 10 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR SIMTEST - - FILTERS STR r,r,i - @EXPTIMES F32 0.1,20.0,20.0 - - NSETUP S32 5 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR SIMTEST - - FILTERS STR r,r,r,r,r - @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 - - NSETUP S32 1 -END - -SEQUENCE METADATA - OBSTYPE STR OBJECT - CAMERA STR SIMTEST - -### Carina - CENTER.RA F32 270.75 - CENTER.DEC F32 -23.7 -### COSMOS field -# CENTER.RA F32 150.119167 -# CENTER.DEC F32 2.205833 - - OFFSET.RA F32 300.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 300.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 3 - OFFSET.ND S32 3 - - DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 3 - DITHER.ND S32 3 - - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest - FILTERS STR r - @EXPTIMES F32 30.00 - @SKYMAGS F32 20.86 - - # XXX use a more realistic IQ distribution... - IQ_MIN F32 0.55 - IQ_MAX F32 1.25 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 - -END Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.megacam.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.megacam.config (revision 43108) +++ (revision ) @@ -1,67 +1,0 @@ - -# define the simulated data to be generated -SEQUENCE MULTI - -SEQUENCE METADATA - OBSTYPE STR BIAS - CAMERA STR MEGACAM - - BIAS.LEVEL F32 100.0 - BIAS.RANGE F32 10.0 - - NIMAGE S32 15 -END - -SEQUENCE METADATA - OBSTYPE STR DARK - CAMERA STR MEGACAM - - DARK.MIN F32 100.0 - DARK.MAX F32 200.0 - - NIMAGE S32 15 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR MEGACAM - - FILTERS STR g,r,i - @EXPTIMES F32 60.0,30.0,30.0 - - NSETUP S32 15 -END - -SEQUENCE_SKIP METADATA - OBSTYPE STR OBJECT - CAMERA STR MEGACAM - - CENTER.RA F32 10.0 - CENTER.DEC F32 60.0 - - OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 5 - OFFSET.ND S32 5 - - DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 2 - DITHER.ND S32 2 - - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.grizy - FILTERS STR g,r,i - @EXPTIMES F32 60.0,30.0,30.0 - @SKYMAGS F32 21.0,20.5,20.0 - - # XXX use a more realistic IQ distribution... - IQ_MIN F32 0.5 - IQ_MAX F32 0.8 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 - -END - Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.nebulous.basic.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.nebulous.basic.auto (revision 43108) +++ (revision ) @@ -1,116 +1,0 @@ - -automate MULTI - -# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK -# success on CHECK -> LAUNCH -# failure on BLOCK -> CHECK - -# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis - -automate METADATA - name STR BIAS - check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" - ncheck S32 20 - launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" - block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" -END - -automate METADATA - name STR DARK - check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" - block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" -END - -automate METADATA - name STR SHUTTER - check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" - block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" -END - -automate METADATA - name STR FLAT-r - check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" - block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" -END - -automate METADATA - name STR FLAT-i - check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -dbname @DBNAME@" - block STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@" -END - -automate METADATA - name STR OBJECT-i - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" - launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" - block STR NONE -END - -automate METADATA - name STR OBJECT-r - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" - launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc" - block STR NONE -END - -### Stack automation??? -automate METADATA - name STR STACK - regular STR "stacktool -definebyquery -all -label proc -workdir neb://anyhost/simtest.@USER@/stack -min_new 4 -min_frac 2 -min_num 4 -select_good_frac_min 0.2 -dbname @DBNAME@" -END - -### Diff automation??? -automate METADATA - name STR DIFF - regular STR "difftool -definewarpstack -label proc -workdir neb://anyhost/simtest.@USER@/diff -good_frac 0.2 -dbname @DBNAME@" -END - -#### Magic automation??? -#automate METADATA -# name STR MAGIC -# regular STR "magictool -definebyquery -workdir neb://anyhost/simtest.@USER@/magic -good_frac 0.2 -dbname @DBNAME@" -#END - -### automate METADATA -### name STR OBJECT-r -### check STR "detselect -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter r" -### launch STR "chiptool -unblock -label simtest-r -dbname @DBNAME@" -### block STR "chiptool -unmasked -label simtest-r -dbname @DBNAME@" -### END - -### automate METADATA -### name STR OBJECT-i -### check STR "detselect -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter i" -### launch STR "chiptool -unblock -label simtest-i -dbname @DBNAME@" -### block STR "chiptool -unmasked -label simtest-i -dbname @DBNAME@" -### END - -### there is a weakness in the label / block business: the labels are -### generic, and the blocks are against those fairly generic words. -### that makes it difficult to block and unblock science exposures -### based on different detrend types that are available. For example, -### to block the r-band against the absence of the desired r-band -### flat, we would need to specify a label specific to the r-band -### images and remove that block when ready. - -### a better approach might be to modify the labels rather than the -### blocks. we can define, eg, chiptool -set_label -definebyquery to -### turn assign the label names based on various properties of the -### interesting images. - -### here is what the automate element might look like for such a circumstance: - -### automate METADATA -### name STR OBJECT-i -### block STR "chiptool -dbname @DBNAME@ -unmasked -label wait -filter i -time_begin 2008/1/1 -time_end 2008/1/2" -### check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter i -time_begin 2008/1/1 -time_end 2008/1/2" -### launch STR "chiptool -dbname @DBNAME@ -set_label proc -label wait -dbname @DBNAME@ -time_begin 2008/1/1 -time_end 2008/1/2" -### END - -### XXX still not quite there.... -### the 'checks' or maybe the 'launch' needs to include a '-n_exp_min' ? -### Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.pro =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.pro (revision 43108) +++ (revision ) @@ -1,289 +1,0 @@ -## simtest.pro : automatic full-suite simulated IPP test : -*- sh -*- - -## there are four features which define a simtest run: -## the camera : any camera can be used, the sim... cameras are defined to be quick tests -## the sequence : this file defines the set of observations generated -## the auto : the automation file defines the sequence of analysis performed -## the recipe : the ppsim recipe defines the features which are included in the generate data - -## these are not completely independent: a certain sequence may be -## required to yield a sensible result for a specific auto, and a -## specific recipe may also be needed - -if ($?PPSIM_RECIPE == 0) - $PPSIM_RECIPE = default -end -if ($?SIMTEST_CAMERA == 0) - $SIMTEST_CAMERA = SIMTEST -end -if ($?SIMTEST_SEQUENCE == 0) - $SIMTEST_SEQUENCE = simtest.basic.config -end -if ($?SIMTEST_AUTO == 0) - $SIMTEST_AUTO = simtest.basic.auto -end -if ($?SIMTEST_THREADS == 0) - $SIMTEST_THREADS = 0 -end - -$SIMTEST_RAWDIR = raw -$SIMTEST_WORKDIR = work - -# these must match the values in $SIMTEST_SEQUENCE above -$CENTER_RA = 270.0 -$CENTER_DEC = 30.0 - -macro simtest - if ($0 != 4) - echo "USAGE: simtest (dbname) (hostname) (init)" - echo " if (init) == run : just run the analysis tasks" - echo " if (init) == inject : (re)create the database and (re)inject the images" - echo " if (init) == new : restart completely from scratch" - echo "" - echo "there are additional options which may be specified by setting the following variable names:" - echo " PPSIM_RECIPE : define the recipe to use when generating the fake data" - echo " SIMTEST_CAMERA : define an alternate camera (otherwise uses entry in sequence file)" - echo " SIMTEST_SEQUENCE : define the set of observations generated (simtest.basic.config)" - echo " SIMTEST_AUTO : define the analysis steps to perform (simtest.basic.auto)" - echo " SIMTEST_THREADS : set the number of threads for the processing node (0)" - echo "" - echo "the following macros can be used to set up specific simtest suites:" - echo " simtest.setup.basic : run standard simtest suite" - echo " simtest.setup.detverify : run detrend creation and detrend verification" - echo " simtest.setup.flatcorr : run a flat-field correction demonstration" - echo " simtest.setup.ctemask : run a ctemask demonstration" - break - end - - $dbname = $1 - $hostname = $2 - $init = $3 - if (("$init" != "run") && ("$init" != "inject") && ("$init" != "new")) - echo "USAGE: simtest (dbname) (hostname) (init)" - echo " if (init) == run : just run the analysis tasks" - echo " if (init) == inject : (re)create the database and (re)inject the images" - echo " if (init) == new : restart completely from scratch" - break - end - - if (("$init" == "new") || ("$init" == "inject")) - # XXX this will fail and exit the script if the db does not exist or is old... - # XXX this step is too dangerous and was eliminated - exec pxadmin -delete -dbname $dbname - - # XXX this gives warnings if the db exists... - # XXX this step is too dangerous and was eliminated - exec pxadmin -create -dbname $dbname - - if ("$init" == "new") - # the labels "wait" and "proc" are special names used in automate.pro - - # confirm the TESS, DVO, and refcat.catdir entries in site.config match the local names - $myTess = `ipp_site_lookup.pl --tess_id TESS` - $myRefcat = `ipp_site_lookup.pl --psastro_db SYNTH.SIMTEST` - - pwd -var myCWD - strsub $myCWD /export /data -var myCWD - - if ("$myTess" != "$myCWD/TESS") - echo "site.config does not list the correct TESS entry: $myTess vs $myCWD/TESS" - break - end - if ("$myRefcat" != "$myCWD/refcat.catdir") - echo "site.config does not list the correct TESS entry: $myTess vs $myCWD/TESS" - break - end - - # make a reference database using refcat.fits - # EAM XXX : 2026.06.29 : this command has a hard-wired RA,DEC region which - # needs to match the object images generated by the SIMTEST_SEQUENCE file - mkref refcat 10 - - ## XXX this will need to be optional as well - ## we could use the book to find OBSTYPE = OBJECT, get CENTER.RA, CENTER.DEC - ## or else we could have ppSimSequence generate an appropriate skycells call - file TESS/Images.dat found - if ($found == 0) - # this command is approriate only to the dither pattern and camera define in simtest.basic.config - exec skycells -mode local -scale 0.2 -center $CENTER_RA $CENTER_DEC -size 0.3 0.3 -nx 3 -ny 3 -D CATDIR TESS - end - - local mySeq - $mySeq = $SIMTEST_SEQUENCE - file $mySeq isFound - if (not($isFound)) - $mySeq = $MODULES:0/$SIMTEST_SEQUENCE - file $mySeq isFound - if (not($isFound)) - echo "cannot find the SIMTEST_SEQUENCE file $SIMTEST_SEQUENCE" - break - end - end - - $ppsim = "ppSimSequence $mySeq simtest.mkimages simtest.inject -basename sim -path $SIMTEST_RAWDIR -workdir $SIMTEST_WORKDIR -dbname $dbname -label wait -dvodb DVO -tess_id TESS -refcat refcat.catdir" - if ("$PPSIM_RECIPE" != "default") - $ppsim = $ppsim -ppsim_recipe $PPSIM_RECIPE - end - $ppsim = $ppsim -camera $SIMTEST_CAMERA - - echo $ppsim - exec $ppsim - - exec source simtest.mkimages - end - - exec chiptool -dbname $dbname -block -label wait - exec source simtest.inject - end - - module pantasks.pro - module automate.pro - - if (1) - module register.pro - module detrend.process.pro - module detrend.stack.pro - module detrend.norm.pro - module detrend.resid.pro - module detrend.reject.pro - module detrend.correct.pro - module chip.pro - module camera.pro - module fake.pro - module warp.pro - module diff.pro - module stack.pro - end - - add.label proc - - if ($SIMTEST_THREADS == 0) - controller host add $hostname - else - controller host add $hostname -threads $SIMTEST_THREADS - end - - add.database $dbname - - automate.load $SIMTEST_AUTO $SIMTEST_CAMERA $dbname - - # module.tasks loads these - if (1) - register.revert.off - detrend.revert.off - chip.revert.off - camera.revert.off - fake.revert.off - warp.revert.off - diff.revert.off - stack.revert.off - end - - run -end - -# XXX add a test function to pxadmin to check for db existence - -## currently defined options for sequence, etc: - -# sequence: simtest.basic.config, simtest.flatcorr.config -# auto: simtest.basic.auto, simtest.flatcorr.auto - -macro simtest.setup.basic - $PPSIM_RECIPE = default - $SIMTEST_SEQUENCE = simtest.basic.config - $SIMTEST_AUTO = simtest.basic.auto -end - -macro simtest.setup.nebulous.basic - $PPSIM_RECIPE = default - $SIMTEST_SEQUENCE = simtest.basic.config - $SIMTEST_AUTO = simtest.nebulous.basic.auto - $SIMTEST_USER = `whoami` - $SIMTEST_RAWDIR = neb://anyhost/simtest.$SIMTEST_USER/raw - $SIMTEST_WORKDIR = neb://anyhost/simtest.$SIMTEST_USER/work -end - -macro simtest.setup.detverify - $PPSIM_RECIPE = default - $SIMTEST_SEQUENCE = simtest.detverify.config - $SIMTEST_AUTO = simtest.detverify.auto -end - -macro simtest.setup.ctemask - $PPSIM_RECIPE = BADCTE.TEST - $SIMTEST_SEQUENCE = simtest.ctemask.config - $SIMTEST_AUTO = simtest.ctemask.auto -end - -macro simtest.setup.flatcorr - $PPSIM_RECIPE = FLATCORR - $SIMTEST_SEQUENCE = simtest.flatcorr.config - $SIMTEST_AUTO = simtest.flatcorr.auto -end - -# basic options for the these images (filter, location, obstype) -$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra $CENTER_RA -dec $CENTER_DEC -pa 0.0 -$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 -$BaseOptions = $BaseOptions -nx 5000 -ny 5000 - -# options for the reference image -$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 -$RefOptions = $RefOptions -D PSF.MODEL PS_MODEL_GAUSS -$RefOptions = $RefOptions -Df STARS.SIGMA.LIM 0.5 -$RefOptions = $RefOptions -Db PSF.CONVOLVE T - -# create a reference database of fake stars to be used by ppSim below -macro mkref - if ($0 != 3) - echo "mkref (refbase) (density)" - break - end - - local refbase - $refbase = $1 - - exec rm -rf $refbase.catdir - exec rm -f $refbase.fits - - $RefOptions = $RefOptions -Df STARS.DENSITY $2 - - # create an image with fake sources and insert the resulting cmf file into a dvodb - $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR $refbase.catdir - - exec ppSim $RefOptions $RefConfig $refbase - - file synth.photcodes found - if (not($found)) - echo "making photcodes file" - mkphotcodes synth.photcodes - end - - exec addstar -D CAMERA simtest -D CATDIR $refbase.catdir -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes $refbase.cmf -quick-airmass - exec relphot -averages -D CATDIR $refbase.catdir -update -region 260 280 -33 -13 -end - -# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: -macro mkphotcodes - if ($0 != 2) - echo "USAGE: mkphotcodes (filename)" - break - end - - exec /bin/rm -f $1 - output $1 - echo "# airmass color astrometry mag photom astrom mask photom mask" - echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" - echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" - echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" - echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" - echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" - echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" - echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" - echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" - echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" - echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" - echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" - output stdout -end - Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.quickmos.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.quickmos.auto (revision 43108) +++ (revision ) @@ -1,74 +1,0 @@ - -automate MULTI - -# stages cycle between states of 'INIT.xxx' and 'RUN.xxx' - -# a stage starts in state INIT.BLOCK. - -# the "automate.block" task runs the 'block' command regularly on -# stages in state INIT.BLOCK. if it returns a valid result, the block -# is set and the stage remains in state INIT.BLOCK. if there is no -# result (ie, no matching BIAS, etc), the stage is put to state -# INIT.CHECK. - -# the "automate.check" tasks runs the 'check' command regularly on -# stages in state INIT.CHECK. if it returns 'ncheck' valid entries -# (default ncheck = 1), the stage is set to state INIT.LAUNCH. - -# the "automate.launch" tasks runs the 'launch' command once on stages -# in state INIT.LAUNCH. after the command is run, the state goes to -# RUN.LAUNCH. After the command has been successfully completed, the -# state goes to DONE.LAUNCH - -# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis - -# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK -# success on CHECK -> LAUNCH -# failure on BLOCK -> CHECK - -automate METADATA - name STR BIAS - check STR "regtool -dbname @DBNAME@ -processedexp -exp_type BIAS -inst @CAMERA@" - ncheck S32 3 - launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS" - block STR "dettool -dbname @DBNAME@ -runs -active -det_type BIAS" -END - -automate METADATA - name STR DARK - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type BIAS -time 2010/01/01" - launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK" - block STR "dettool -dbname @DBNAME@ -runs -active -det_type DARK" -END - -automate METADATA - name STR FLAT-r - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type DARK -time 2010/01/01" - launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r" - block STR "dettool -dbname @DBNAME@ -runs -active -det_type FLAT -filter r" -END - -automate METADATA - name STR OBJECT-r - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -filter r -time 2010/01/01" - launch STR "chiptool -dbname @DBNAME@ -definebyquery -inst @CAMERA@ -filter r -set_label proc -set_data_group simtest_r -set_dist_group simtest -set_workdir file://@CWD@/chip" - block STR "chiptool -dbname @DBNAME@ -listrun -filter r -label proc" -END - -### Stack automation -automate METADATA - name STR STACK - check STR "warptool -dbname @DBNAME@ -listrun -filter r -label proc -state full" - ncheck S32 6 - launch STR "stacktool -dbname @DBNAME@ -definebyquery -select_filter r -set_label proc -set_data_group simtest_r -set_dist_group simtest -set_workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -min_num 4" - block STR "stacktool -dbname @DBNAME@ -listrun -filter r -label proc" -END - -### Diff automation -automate METADATA - name STR DIFF - check STR "stacktool -dbname @DBNAME@ -listrun -filter r -label proc -state full" - ncheck S32 9 - launch STR "difftool -dbname @DBNAME@ -definewarpstack -filter r -stack_label proc -warp_label proc -set_label proc -set_data_group simtest_r -set_dist_group simtest -set_workdir file://@CWD@/diff -good_frac 0.2" - block STR "difftool -dbname @DBNAME@ -listrun -filter r -label proc" -END Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.quickmos.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.quickmos.config (revision 43108) +++ (revision ) @@ -1,62 +1,0 @@ - -CAMERA STR SIMMOSAIC - -# allows for multi-file outputs to be selected by ppSimSequence -FILERULE STR *fits - -# define the simulated data to be generated -SEQUENCE MULTI - -SEQUENCE METADATA - OBSTYPE STR BIAS - NIMAGES S32 3 -END - -SEQUENCE METADATA - OBSTYPE STR DARK - @EXPTIMES F32 30.0, 300.0 - @NIMAGES S32 2, 2 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - - FILTERS STR r - @EXPTIMES F32 30.0 - - NSETUP S32 3 -END - -SEQUENCE METADATA - OBSTYPE STR OBJECT - - CENTER.RA F32 270.0 - CENTER.DEC F32 30.0 - - OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 1 - OFFSET.ND S32 1 - - DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 3 - DITHER.ND S32 2 - - DVODB STR NONE - FILTERS STR r - @EXPTIMES F32 10.00 - @SKYMAGS F32 20.86 - - # XXX use a more realistic IQ distribution... - IQ_MIN F32 0.9 - IQ_MAX F32 1.2 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 - - MJD_MIN F32 60000.0 - MJD_MAX F32 60010.0 -END Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.simmosaic.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.simmosaic.config (revision 43108) +++ (revision ) @@ -1,110 +1,0 @@ - -CAMERA STR SIMMOSAIC - -FILERULE STR *.fits - -# define the simulated data to be generated -SEQUENCE MULTI - -SEQUENCE METADATA - OBSTYPE STR BIAS - NIMAGES S32 20 -END - -SEQUENCE METADATA - OBSTYPE STR DARK - @EXPTIMES F32 30.0, 300.0 - @NIMAGES S32 10, 10 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - - FILTERS STR r,r,i - @EXPTIMES F32 0.1,20.0,20.0 - - NSETUP S32 5 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - - FILTERS STR r,r,r,r,r - @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 - - NSETUP S32 1 -END - -SEQUENCE METADATA - OBSTYPE STR OBJECT - -### Carina - CENTER.RA F32 270.75 - CENTER.DEC F32 -23.7 -### COSMOS field -# CENTER.RA F32 150.119167 -# CENTER.DEC F32 2.205833 - - OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 1 - OFFSET.ND S32 1 - - DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 3 - DITHER.ND S32 2 - - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest - FILTERS STR r,i - @EXPTIMES F32 10.00,10.00 - @SKYMAGS F32 20.86,20.15 - - # XXX use a more realistic IQ distribution... - IQ_MIN F32 0.55 - IQ_MAX F32 1.25 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 - -END - - -SEQUENCE METADATA - OBSTYPE STR OBJECT - -### Carina - CENTER.RA F32 270.75 - CENTER.DEC F32 -23.7 -### COSMOS field -# CENTER.RA F32 150.119167 -# CENTER.DEC F32 2.205833 - - OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 1 - OFFSET.ND S32 1 - - DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 1 - DITHER.ND S32 1 - - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest - FILTERS STR r,r,i,i - @EXPTIMES F32 5.0, 240.0, 5.0, 240.0 - @SKYMAGS F32 20.86,20.86,20.15,20.15 - - # XXX use a more realistic IQ distribution... - IQ_MIN F32 0.55 - IQ_MAX F32 1.25 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 - -END - Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.stack.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.stack.auto (revision 43108) +++ (revision ) @@ -1,65 +1,0 @@ - -automate MULTI - -# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK -# success on CHECK -> LAUNCH -# failure on BLOCK -> CHECK - -# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis - -automate METADATA - name STR BIAS - check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" - ncheck S32 20 - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" - block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" -END - -automate METADATA - name STR DARK - check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" - block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" -END - -automate METADATA - name STR SHUTTER - check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" - block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" -END - -automate METADATA - name STR FLAT-r - check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" - block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" -END - -automate METADATA - name STR FLAT-i - check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" - launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -dbname @DBNAME@" - block STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@" -END - -automate METADATA - name STR OBJECT-i - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" - launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" - block STR NONE -END - -automate METADATA - name STR OBJECT-r - check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" - launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc" - block STR NONE -END - -### Stack automation??? -automate METADATA - name STR STACK - regular STR "stacktool -definebyquery -label proc -workdir file://@CWD@/stack -min_new 4 -min_num 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@" -END - Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.stack.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/simtest.stack.config (revision 43108) +++ (revision ) @@ -1,76 +1,0 @@ - -# define the simulated data to be generated -SEQUENCE MULTI - -SEQUENCE METADATA - OBSTYPE STR BIAS - CAMERA STR SIMTEST - NIMAGES S32 20 -END - -SEQUENCE METADATA - OBSTYPE STR DARK - CAMERA STR SIMTEST - @EXPTIMES F32 30.0, 300.0 - @NIMAGES S32 10, 10 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR SIMTEST - - FILTERS STR r,r - @EXPTIMES F32 0.1,20.0 - - NSETUP S32 5 -END - -SEQUENCE METADATA - OBSTYPE STR FLAT - CAMERA STR SIMTEST - - FILTERS STR r,r,r,r,r - @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 - - NSETUP S32 1 -END - - -SEQUENCE METADATA -### We want both short and long exposures, and dark and bright sky - OBSTYPE STR OBJECT - CAMERA STR SIMTEST - -### Carina - CENTER.RA F32 270.75 - CENTER.DEC F32 -23.7 -### COSMOS field -# CENTER.RA F32 150.119167 -# CENTER.DEC F32 2.205833 - - OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) - OFFSET.NR S32 1 - OFFSET.ND S32 1 - - DITHER.RA F32 10.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.DEC F32 10.0 # linear offset in arcsec (do not include cos(DEC) correction) - DITHER.NR S32 3 - DITHER.ND S32 3 - - DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest - FILTERS STR r,r,r - @EXPTIMES F32 10,300,300 - @SKYMAGS F32 21,21,18 - - # XXX use a more realistic IQ distribution... - IQ_MIN F32 1.00 - IQ_MAX F32 1.00 - - # rotation sequence - POS_MIN F32 0.0 - POS_MAX F32 0.0 - POS_DELTA F32 1.0 - -END - Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/site.manoa.pro =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/site.manoa.pro (revision 43108) +++ (revision ) @@ -1,52 +1,0 @@ -## site.mhpcc.pro : example configuration script for the MHPCC IPP cluster : -*- sh -*- - -## a site configuration needs to define the following pieces of information: -## 1) the machines used for processing -## 2) the default_host and workdir_template variables -## 3) the ipphost table - -macro init.cluster.po - $PARALLEL = 1 - controller exit true - - # po05 -- using for pantasks - # po23 -- broken perl - - controller host add po02 - controller host add po03 - controller host add po04 -# controller host add po05 - controller host add po06 - controller host add po07 -# controller host add po08 -# controller host add po09 -# controller host add po10 -# controller host add po11 -# controller host add po12 -# controller host add po13 -# controller host add po14 -# controller host add po15 -# controller host add po16 -# controller host add po17 -# controller host add po18 -# controller host add po19 -# controller host add po20 -# controller host add po21 -# controller host add po22 -# controller host add po23 -# controller host add po24 -end - -macro init.cluster.sn - $PARALLEL = 1 - controller exit true - controller host add sn2 - controller host add sn3 - controller host add sn4 - controller host add sn5 -end - -macro init.site - queueload tmp -x "cat $MODULES:0/ipphosts.manoa.config" - ipptool2book tmp ipphosts -key camera -end Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/site.mhpcc.pro =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/site.mhpcc.pro (revision 43108) +++ (revision ) @@ -1,98 +1,0 @@ -## site.mhpcc.pro : example configuration script for the MHPCC IPP cluster : -*- sh -*- - -echo "do not use this configuration file (site.mhpcc.pro)" -break - -## a site configuration needs to define the following pieces of information: -## 1) the machines used for processing -## 2) the default_host and workdir_template variables -## 3) the ipphost table - -macro init.cluster.mhpcc - $PARALLEL = 1 - controller exit true - -# controller host add ipp014 - controller host add ipp021 - controller host add ipp015 - controller host add ipp023 - controller host add ipp024 - controller host add ipp025 - controller host add ipp026 - controller host add ipp028 - controller host add ipp029 - controller host add ipp030 - controller host add ipp031 - controller host add ipp032 - controller host add ipp033 - controller host add ipp034 - controller host add ipp035 - controller host add ipp036 - controller host add ipp038 - controller host add ipp039 - controller host add ipp040 - controller host add ipp041 - controller host add ipp042 - controller host add ipp043 - controller host add ipp044 - controller host add ipp045 - controller host add ipp046 - controller host add ipp047 - controller host add ipp048 - controller host add ipp049 - controller host add ipp050 - controller host add ipp051 - controller host add ipp052 -# controller host add ipp053 -end - -## override the basic inits set in pantasks.pro -macro init.copy.mhpcc - if ($0 != 2) - echo "USAGE: init.copy.mhpcc (nebulous)" - echo "nebulous may be 'on' or 'off'" - break - end - - if (("$1" != "on") && ("$1" != "off")) - echo "USAGE: init.copy.mhpcc (nebulous)" - echo "nebulous may be 'on' or 'off'" - break - end - - # XXX this is only used by summit.copy.pro. move this as a check into summit.copy.pro? - $COMPRESS = 1 - - # the templates are used if we have a class_id/host relationship; - # if none is found, the default values are used - # XXX not sure how to handle the .N value if we need to use more than one - - if ("$1" == "on") - $NEBULOUS = 1 - $default_host = any - $workdir_template = neb://@HOST@.0 - else - $NEBULOUS = 0 - $default_host = ipp023 - $workdir_template = /data/@HOST@.0 - end -end - -macro init.site - init.cluster.mhpcc - init.copy.mhpcc on - - queueload tmp -x "cat $MODULES:0/ipphosts.mhpcc.config" - ipptool2book tmp ipphosts -key camera - - # set autoprocessing for chip - queueload tmp -x "cat $MODULES:0/surveys.mhpcc.config" - ipptool2book tmp surveys -key survey -end - -macro init.site.nohosts - init.copy.mhpcc on - - queueload tmp -x "cat $MODULES:0/ipphosts.mhpcc.config" - ipptool2book tmp ipphosts -key camera -end Index: anches/eam_branches/ipp-pstamp-20260421/ippTasks/surveys.mhpcc.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/ippTasks/surveys.mhpcc.config (revision 43108) +++ (revision ) @@ -1,70 +1,0 @@ -# Note from Ken Chambers on the OBS_MODE keyword: -# OBS_MODE - Official Keyword - string use for Survey ID, 3PI, MD, SS, STS, M31, CAL, AF1, AF2, AF3, ENG, NA - from scheduler - -surveys MULTI -surveys METADATA - camera STR isp - survey STR NULL - label STR @CAMERA@.@DATE@ - dvodb STR /data/ipp004.0/isp/catdir.isp - worksubdir STR neb://@HOST@.0/isp/@DATE@ - end_stage STR camera - tess_id STR RINGS.V0 -END - - - -#all of these are really old - never use them, except maybe as an example? -#surveys METADATA -# survey STR 3PI -# label STR ThreePi.@CAMERA@.@FILTER@.@DATE@ -# dvodb STR /data/ipp004.0/gpc1/catdir.ThreePi -# worksubdir STR ThreePi -# end_stage STR warp -# tess_id STR ALLSKY -#END - -#surveys METADATA -# survey STR MD -# label STR MediumDeep.@CAMERA@.@FILTER@.@DATE@ -# dvodb STR /data/ipp004.0/gpc1/MediumDeep -# worksubdir STR MediumDeep -# end_stage STR warp -# tess_id STR ALLSKY -#END - -#surveys METADATA -# survey STR SS -# label STR SolarSystem.@CAMERA@.@FILTER@.@DATE@ -# dvodb STR /data/ipp004.0/gpc1/SolarSystem -# worksubdir STR SolarSystem -# end_stage STR warp -# tess_id STR ALLSKY -#END - -#surveys METADATA -# survey STR STS -# label STR STS.@CAMERA@.@FILTER@.@DATE@ -# dvodb STR /data/ipp004.0/gpc1/STS -# worksubdir STR STS -# end_stage STR warp -# tess_id STR ALLSKY -#END - -#surveys METADATA -# survey STR M31 -# label STR M31.@CAMERA@.@FILTER@.@DATE@ -# dvodb STR /data/ipp004.0/gpc1/M31 -# worksubdir STR M31 -# end_stage STR warp -# tess_id STR ALLSKY -#END - -#surveys METADATA -# survey STR CAL -# label STR CalFields.@CAMERA@.@FILTER@.@DATE@ -# dvodb STR /data/ipp004.0/gpc1/CalFields -# worksubdir STR CalFields -# end_stage STR warp -# tess_id STR ALLSKY -#END Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.basic.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.basic.auto (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.basic.auto (revision 43109) @@ -0,0 +1,128 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# a stage starts in state INIT.BLOCK. + +# the "automate.block" task runs the 'block' command regularly. if it +# returns a valid result, the block is set and the state remains in +# block. if there is no result (ie, no matching BIAS, etc), the stage +# is put to state INIT.CHECK. + +# the "automate.check" tasks runs the 'check' command regularly. if +# it returns ncheck valid entries (default ncheck = 1), the state goes +# to INIT.LAUNCH. + +# the "automate.launch" tasks runs the 'launch' command once. if it returns +# ncheck valid entries (default ncheck = 1), the state goes to +# INIT.LAUNCH. + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR BIAS + check STR "regtool -dbname @DBNAME@ -processedexp -exp_type BIAS -inst @CAMERA@" + ncheck S32 20 + launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS" + block STR "dettool -dbname @DBNAME@ -runs -active -det_type BIAS" +END + +automate METADATA + name STR DARK + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type BIAS -time 2010/01/01" + launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK" + block STR "dettool -dbname @DBNAME@ -runs -active -det_type DARK" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type DARK -time 2010/01/01" + launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r" + block STR "dettool -dbname @DBNAME@ -runs -active -det_type SHUTTER" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type SHUTTER -time 2010/01/01" + launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r" + block STR "dettool -dbname @DBNAME@ -runs -active -det_type FLAT -filter r" +END + +automate METADATA + name STR FLAT-i + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type SHUTTER -time 2010/01/01" + launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i" + block STR "dettool -dbname @DBNAME@ -runs -active -det_type FLAT -filter i" +END + +## the lines below assume the simtest suite injects the chips with an end state of chip, but currently they only go to 'reg' +automate METADATA + name STR OBJECT-i + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -filter i -time 2010/01/01" +# launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -label wait -filter i -set_label proc -set_data_group simtest_i -set_dist_group simtest" + launch STR "chiptool -dbname @DBNAME@ -definebyquery -inst @CAMERA@ -filter i -set_label proc -set_data_group simtest_i -set_dist_group simtest -set_workdir file://@CWD@/chip" + block STR "chiptool -dbname @DBNAME@ -listrun -filter i -label proc" +END + +automate METADATA + name STR OBJECT-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -filter r -time 2010/01/01" +# launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -label wait -filter r -set_label proc -set_data_group simtest_r -set_dist_group simtest" + launch STR "chiptool -dbname @DBNAME@ -definebyquery -inst @CAMERA@ -filter r -set_label proc -set_data_group simtest_i -set_dist_group simtest -set_workdir file://@CWD@/chip" + block STR "chiptool -dbname @DBNAME@ -listrun -filter r -label proc" +END + +### ### Stack automation +### automate METADATA +### name STR STACK +### regular STR "stacktool -dbname @DBNAME@ -definebyquery -set_label proc -set_data_group simtest_r -set_dist_group simtest -set_workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -min_num 4" +### END +### +### ### Diff automation +### automate METADATA +### name STR DIFF +### regular STR "difftool -dbname @DBNAME@ -definewarpstack -set_label proc -set_workdir file://@CWD@/diff -good_frac 0.2" +### END + +### automate METADATA +### name STR OBJECT-r +### check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -filter r" +### launch STR "chiptool -dbname @DBNAME@ -unblock -label simtest-r" +### block STR "chiptool -dbname @DBNAME@ -unmasked -label simtest-r" +### END + +### automate METADATA +### name STR OBJECT-i +### check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -filter i" +### launch STR "chiptool -dbname @DBNAME@ -unblock -label simtest-i" +### block STR "chiptool -dbname @DBNAME@ -unmasked -label simtest-i" +### END + +### there is a weakness in the label / block business: the labels are +### generic, and the blocks are against those fairly generic words. +### that makes it difficult to block and unblock science exposures +### based on different detrend types that are available. For example, +### to block the r-band against the absence of the desired r-band +### flat, we would need to specify a label specific to the r-band +### images and remove that block when ready. + +### a better approach might be to modify the labels rather than the +### blocks. we can define, eg, chiptool -set_label -definebyquery to +### turn assign the label names based on various properties of the +### interesting images. + +### here is what the automate element might look like for such a circumstance: + +### automate METADATA +### name STR OBJECT-i +### block STR "chiptool -dbname @DBNAME@ -unmasked -label wait -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### launch STR "chiptool -dbname @DBNAME@ -set_label proc -label wait -time_begin 2008/1/1 -time_end 2008/1/2" +### END + +### XXX still not quite there.... +### the 'checks' or maybe the 'launch' needs to include a '-n_exp_min' ? +### Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.basic.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.basic.config (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.basic.config (revision 43109) @@ -0,0 +1,115 @@ + +# allows for multi-file outputs to be selected by ppSimSequence +FILERULE STR *fits + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 1 + DITHER.ND S32 1 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,r,i,i + @EXPTIMES F32 5.0, 240.0, 5.0, 240.0 + @SKYMAGS F32 20.86,20.86,20.15,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.ctemask.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.ctemask.auto (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.ctemask.auto (revision 43109) @@ -0,0 +1,39 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 5 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" +END + +# XXX let's build the necessary data, then see how to run the cte mask generation (input is flattened flats) Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.ctemask.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.ctemask.config (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.ctemask.config (revision 43109) @@ -0,0 +1,38 @@ + +# very basic test to generate flats and test the CTE masking process +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 3, 3 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r + @EXPTIMES F32 20.0 + + NSETUP S32 7 +END + Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.det =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.det (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.det (revision 43109) @@ -0,0 +1,42 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type BIAS -select_exp_type BIAS" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK + prereq STR BIAS + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR DARK + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i" + dbname STR eamtest + pantaskState STR NULL +END Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.detverify.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.detverify.auto (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.detverify.auto (revision 43109) @@ -0,0 +1,78 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 20 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR BIAS_VERIFY + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -mode verify -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR DARK_VERIFY + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -mode verify -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER_VERIFY + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -mode verify -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-i + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT_VERIFY-r + check STR "detselect -search -inst @CAMERA@ -det_type FLAT -filter r -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -mode verify -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT_VERIFY-i + check STR "detselect -search -inst @CAMERA@ -det_type FLAT -filter i -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -mode verify -dbname @DBNAME@" +END + Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.detverify.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.detverify.config (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.detverify.config (revision 43109) @@ -0,0 +1,37 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.flatcorr.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.flatcorr.auto (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.flatcorr.auto (revision 43109) @@ -0,0 +1,76 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# this is a simplified automate script which only models BIAS and FLAT. +# generate appropriate simulated data with the FLATCORR recipe + +# wait until 20 bias frames are registered, then define a new bias run +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 10 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +# wait until the master bias frame is registered, then define a new dark run +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +# wait until the master dark frame is registered, then define a new shutter run +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +# wait until the master bias frame is registered, then define a new flat-r run +automate METADATA + name STR RAWFLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT_RAW -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT_RAW -filter r -dbname @DBNAME@" +END + +# wait until the master flat-r frame is registered, then initiate a flat-field correction analysis +automate METADATA + name STR FLATCORR-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT_RAW" + launch STR "flatcorr -definebyquery -dbname @DBNAME@ -det_type FLATCORR -inst @CAMERA@ -exp_type OBJECT -filter r -set_workdir file://@CWD@/detwork -set_reduction FLATCORR -set_dvodb @CWD@/catdir.flatcorr -set_region 270.6,270.9:-23.8,-23.5 -set_filter r" + block STR "dettool -runs -active -det_type FLATCORR -filter r -dbname @DBNAME@" +END + +# wait until the flat-r correction frame is registered, then define the flat-fields to be corrected +# XXX for the moment, the det_id here is hard-wired +automate METADATA + name STR FLATFIX-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLATCORR" + launch STR "dettool -makecorrection -dbname @DBNAME@ -det_id 4 -iteration 0 -det_type FLAT -workdir file://@CWD@/detwork" + block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" +END + +# wait until the corrected master flat-r frame is registered, then initiate a second flat-field correction analysis +automate METADATA + name STR FLATTEST-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" + launch STR "flatcorr -definebyquery -dbname @DBNAME@ -det_type FLATTEST -inst @CAMERA@ -exp_type OBJECT -filter r -set_workdir file://@CWD@/detwork -set_reduction FLATTEST -set_dvodb @CWD@/catdir.flattest -set_region 270.6,270.9:-23.8,-23.5 -set_filter r" + block STR "dettool -runs -active -det_type FLATCORR -filter r -dbname @DBNAME@" +END + +## # wait until the master flat-r frame is registered, then unblock the r-band data for chip-level processing +## automate METADATA +## name STR OBJECT-i +## check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" +## launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" +## block STR NONE +## END +## Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.flatcorr.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.flatcorr.config (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.flatcorr.config (revision 43109) @@ -0,0 +1,189 @@ + +# generate a simple fake data set with 2x2 dithers at offset of +# 50, 100, 200 arcsec (camera is 530 arcsec on a side) + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 10 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 5, 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r + @EXPTIMES F32 0.1,20.0 + + NSETUP S32 3 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +# a single image at the field center +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + + ### Carina (modestly high density) + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 + + # the simtest camera is 2k x 2k, 530 arcsec on a side + # XXX note that we cannot define the offsets in terms of the camera dimensions... + + OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 1 + DITHER.ND S32 1 + + # XXX is DVODB used? + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r + @EXPTIMES F32 10.00 + @SKYMAGS F32 20.86 + + # modest IQ range here + IQ_MIN F32 0.70 + IQ_MAX F32 0.90 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + +# dither set @ scale 1 = 60 arcsec +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + + ### Carina (modestly high density) + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 + + # XXX note that we cannot define the offsets in terms of the camera dimensions... + OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + # XXX is DVODB used? + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r + @EXPTIMES F32 10.00 + @SKYMAGS F32 20.86 + + # modest IQ range here + IQ_MIN F32 0.70 + IQ_MAX F32 0.90 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + +# dither set @ scale 2 = 120 arcsec +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + + ### Carina (modestly high density) + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 + + # XXX note that we cannot define the offsets in terms of the camera dimensions... + OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 120.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 120.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + # XXX is DVODB used? + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r + @EXPTIMES F32 10.00 + @SKYMAGS F32 20.86 + + # modest IQ range here + IQ_MIN F32 0.70 + IQ_MAX F32 0.90 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + +# dither set @ scale 3 = 240 arcsec +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + + ### Carina (modestly high density) + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 + + # XXX note that we cannot define the offsets in terms of the camera dimensions... + OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 240.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 240.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + # XXX is DVODB used? + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r + @EXPTIMES F32 10.00 + @SKYMAGS F32 20.86 + + # modest IQ range here + IQ_MIN F32 0.70 + IQ_MAX F32 0.90 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.images.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.images.auto (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.images.auto (revision 43109) @@ -0,0 +1,22 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR OBJECT-i + check STR "regtool -processedexp -exp_type OBJECT -filter i -inst @CAMERA@ -dbname @DBNAME@" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc -set_data_group simtest_i -set_dist_group simtest" + block STR NONE +END + +automate METADATA + name STR OBJECT-r + check STR "regtool -processedexp -exp_type OBJECT -filter r -inst @CAMERA@ -dbname @DBNAME@" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc -set_data_group simtest_r -set_dist_group simtest" + block STR NONE +END Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.images.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.images.config (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.images.config (revision 43109) @@ -0,0 +1,77 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 +END + + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 1 + DITHER.ND S32 1 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,r,i,i + @EXPTIMES F32 5.0, 240.0, 5.0, 240.0 + @SKYMAGS F32 20.86,20.86,20.15,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 +END + Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.large.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.large.config (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.large.config (revision 43109) @@ -0,0 +1,73 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 300.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 300.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 3 + OFFSET.ND S32 3 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 3 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r + @EXPTIMES F32 30.00 + @SKYMAGS F32 20.86 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.megacam.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.megacam.config (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.megacam.config (revision 43109) @@ -0,0 +1,67 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR MEGACAM + + BIAS.LEVEL F32 100.0 + BIAS.RANGE F32 10.0 + + NIMAGE S32 15 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR MEGACAM + + DARK.MIN F32 100.0 + DARK.MAX F32 200.0 + + NIMAGE S32 15 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR MEGACAM + + FILTERS STR g,r,i + @EXPTIMES F32 60.0,30.0,30.0 + + NSETUP S32 15 +END + +SEQUENCE_SKIP METADATA + OBSTYPE STR OBJECT + CAMERA STR MEGACAM + + CENTER.RA F32 10.0 + CENTER.DEC F32 60.0 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 5 + OFFSET.ND S32 5 + + DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.grizy + FILTERS STR g,r,i + @EXPTIMES F32 60.0,30.0,30.0 + @SKYMAGS F32 21.0,20.5,20.0 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.5 + IQ_MAX F32 0.8 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.nebulous.basic.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.nebulous.basic.auto (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.nebulous.basic.auto (revision 43109) @@ -0,0 +1,116 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 20 + launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-i + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@" +END + +automate METADATA + name STR OBJECT-i + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" + block STR NONE +END + +automate METADATA + name STR OBJECT-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc" + block STR NONE +END + +### Stack automation??? +automate METADATA + name STR STACK + regular STR "stacktool -definebyquery -all -label proc -workdir neb://anyhost/simtest.@USER@/stack -min_new 4 -min_frac 2 -min_num 4 -select_good_frac_min 0.2 -dbname @DBNAME@" +END + +### Diff automation??? +automate METADATA + name STR DIFF + regular STR "difftool -definewarpstack -label proc -workdir neb://anyhost/simtest.@USER@/diff -good_frac 0.2 -dbname @DBNAME@" +END + +#### Magic automation??? +#automate METADATA +# name STR MAGIC +# regular STR "magictool -definebyquery -workdir neb://anyhost/simtest.@USER@/magic -good_frac 0.2 -dbname @DBNAME@" +#END + +### automate METADATA +### name STR OBJECT-r +### check STR "detselect -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter r" +### launch STR "chiptool -unblock -label simtest-r -dbname @DBNAME@" +### block STR "chiptool -unmasked -label simtest-r -dbname @DBNAME@" +### END + +### automate METADATA +### name STR OBJECT-i +### check STR "detselect -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter i" +### launch STR "chiptool -unblock -label simtest-i -dbname @DBNAME@" +### block STR "chiptool -unmasked -label simtest-i -dbname @DBNAME@" +### END + +### there is a weakness in the label / block business: the labels are +### generic, and the blocks are against those fairly generic words. +### that makes it difficult to block and unblock science exposures +### based on different detrend types that are available. For example, +### to block the r-band against the absence of the desired r-band +### flat, we would need to specify a label specific to the r-band +### images and remove that block when ready. + +### a better approach might be to modify the labels rather than the +### blocks. we can define, eg, chiptool -set_label -definebyquery to +### turn assign the label names based on various properties of the +### interesting images. + +### here is what the automate element might look like for such a circumstance: + +### automate METADATA +### name STR OBJECT-i +### block STR "chiptool -dbname @DBNAME@ -unmasked -label wait -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### launch STR "chiptool -dbname @DBNAME@ -set_label proc -label wait -dbname @DBNAME@ -time_begin 2008/1/1 -time_end 2008/1/2" +### END + +### XXX still not quite there.... +### the 'checks' or maybe the 'launch' needs to include a '-n_exp_min' ? +### Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.pro =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.pro (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.pro (revision 43109) @@ -0,0 +1,289 @@ +## simtest.pro : automatic full-suite simulated IPP test : -*- sh -*- + +## there are four features which define a simtest run: +## the camera : any camera can be used, the sim... cameras are defined to be quick tests +## the sequence : this file defines the set of observations generated +## the auto : the automation file defines the sequence of analysis performed +## the recipe : the ppsim recipe defines the features which are included in the generate data + +## these are not completely independent: a certain sequence may be +## required to yield a sensible result for a specific auto, and a +## specific recipe may also be needed + +if ($?PPSIM_RECIPE == 0) + $PPSIM_RECIPE = default +end +if ($?SIMTEST_CAMERA == 0) + $SIMTEST_CAMERA = SIMTEST +end +if ($?SIMTEST_SEQUENCE == 0) + $SIMTEST_SEQUENCE = simtest.basic.config +end +if ($?SIMTEST_AUTO == 0) + $SIMTEST_AUTO = simtest.basic.auto +end +if ($?SIMTEST_THREADS == 0) + $SIMTEST_THREADS = 0 +end + +$SIMTEST_RAWDIR = raw +$SIMTEST_WORKDIR = work + +# these must match the values in $SIMTEST_SEQUENCE above +$CENTER_RA = 270.0 +$CENTER_DEC = 30.0 + +macro simtest + if ($0 != 4) + echo "USAGE: simtest (dbname) (hostname) (init)" + echo " if (init) == run : just run the analysis tasks" + echo " if (init) == inject : (re)create the database and (re)inject the images" + echo " if (init) == new : restart completely from scratch" + echo "" + echo "there are additional options which may be specified by setting the following variable names:" + echo " PPSIM_RECIPE : define the recipe to use when generating the fake data" + echo " SIMTEST_CAMERA : define an alternate camera (otherwise uses entry in sequence file)" + echo " SIMTEST_SEQUENCE : define the set of observations generated (simtest.basic.config)" + echo " SIMTEST_AUTO : define the analysis steps to perform (simtest.basic.auto)" + echo " SIMTEST_THREADS : set the number of threads for the processing node (0)" + echo "" + echo "the following macros can be used to set up specific simtest suites:" + echo " simtest.setup.basic : run standard simtest suite" + echo " simtest.setup.detverify : run detrend creation and detrend verification" + echo " simtest.setup.flatcorr : run a flat-field correction demonstration" + echo " simtest.setup.ctemask : run a ctemask demonstration" + break + end + + $dbname = $1 + $hostname = $2 + $init = $3 + if (("$init" != "run") && ("$init" != "inject") && ("$init" != "new")) + echo "USAGE: simtest (dbname) (hostname) (init)" + echo " if (init) == run : just run the analysis tasks" + echo " if (init) == inject : (re)create the database and (re)inject the images" + echo " if (init) == new : restart completely from scratch" + break + end + + if (("$init" == "new") || ("$init" == "inject")) + # XXX this will fail and exit the script if the db does not exist or is old... + # XXX this step is too dangerous and was eliminated + exec pxadmin -delete -dbname $dbname + + # XXX this gives warnings if the db exists... + # XXX this step is too dangerous and was eliminated + exec pxadmin -create -dbname $dbname + + if ("$init" == "new") + # the labels "wait" and "proc" are special names used in automate.pro + + # confirm the TESS, DVO, and refcat.catdir entries in site.config match the local names + $myTess = `ipp_site_lookup.pl --tess_id TESS` + $myRefcat = `ipp_site_lookup.pl --psastro_db SYNTH.SIMTEST` + + pwd -var myCWD + strsub $myCWD /export /data -var myCWD + + if ("$myTess" != "$myCWD/TESS") + echo "site.config does not list the correct TESS entry: $myTess vs $myCWD/TESS" + break + end + if ("$myRefcat" != "$myCWD/refcat.catdir") + echo "site.config does not list the correct TESS entry: $myTess vs $myCWD/TESS" + break + end + + # make a reference database using refcat.fits + # EAM XXX : 2026.06.29 : this command has a hard-wired RA,DEC region which + # needs to match the object images generated by the SIMTEST_SEQUENCE file + mkref refcat 10 + + ## XXX this will need to be optional as well + ## we could use the book to find OBSTYPE = OBJECT, get CENTER.RA, CENTER.DEC + ## or else we could have ppSimSequence generate an appropriate skycells call + file TESS/Images.dat found + if ($found == 0) + # this command is approriate only to the dither pattern and camera define in simtest.basic.config + exec skycells -mode local -scale 0.2 -center $CENTER_RA $CENTER_DEC -size 0.3 0.3 -nx 3 -ny 3 -D CATDIR TESS + end + + local mySeq + $mySeq = $SIMTEST_SEQUENCE + file $mySeq isFound + if (not($isFound)) + $mySeq = $MODULES:0/$SIMTEST_SEQUENCE + file $mySeq isFound + if (not($isFound)) + echo "cannot find the SIMTEST_SEQUENCE file $SIMTEST_SEQUENCE" + break + end + end + + $ppsim = "ppSimSequence $mySeq simtest.mkimages simtest.inject -basename sim -path $SIMTEST_RAWDIR -workdir $SIMTEST_WORKDIR -dbname $dbname -label wait -dvodb DVO -tess_id TESS -refcat refcat.catdir" + if ("$PPSIM_RECIPE" != "default") + $ppsim = $ppsim -ppsim_recipe $PPSIM_RECIPE + end + $ppsim = $ppsim -camera $SIMTEST_CAMERA + + echo $ppsim + exec $ppsim + + exec source simtest.mkimages + end + + exec chiptool -dbname $dbname -block -label wait + exec source simtest.inject + end + + module pantasks.pro + module automate.pro + + if (1) + module register.pro + module detrend.process.pro + module detrend.stack.pro + module detrend.norm.pro + module detrend.resid.pro + module detrend.reject.pro + module detrend.correct.pro + module chip.pro + module camera.pro + module fake.pro + module warp.pro + module diff.pro + module stack.pro + end + + add.label proc + + if ($SIMTEST_THREADS == 0) + controller host add $hostname + else + controller host add $hostname -threads $SIMTEST_THREADS + end + + add.database $dbname + + automate.load $SIMTEST_AUTO $SIMTEST_CAMERA $dbname + + # module.tasks loads these + if (1) + register.revert.off + detrend.revert.off + chip.revert.off + camera.revert.off + fake.revert.off + warp.revert.off + diff.revert.off + stack.revert.off + end + + run +end + +# XXX add a test function to pxadmin to check for db existence + +## currently defined options for sequence, etc: + +# sequence: simtest.basic.config, simtest.flatcorr.config +# auto: simtest.basic.auto, simtest.flatcorr.auto + +macro simtest.setup.basic + $PPSIM_RECIPE = default + $SIMTEST_SEQUENCE = simtest.basic.config + $SIMTEST_AUTO = simtest.basic.auto +end + +macro simtest.setup.nebulous.basic + $PPSIM_RECIPE = default + $SIMTEST_SEQUENCE = simtest.basic.config + $SIMTEST_AUTO = simtest.nebulous.basic.auto + $SIMTEST_USER = `whoami` + $SIMTEST_RAWDIR = neb://anyhost/simtest.$SIMTEST_USER/raw + $SIMTEST_WORKDIR = neb://anyhost/simtest.$SIMTEST_USER/work +end + +macro simtest.setup.detverify + $PPSIM_RECIPE = default + $SIMTEST_SEQUENCE = simtest.detverify.config + $SIMTEST_AUTO = simtest.detverify.auto +end + +macro simtest.setup.ctemask + $PPSIM_RECIPE = BADCTE.TEST + $SIMTEST_SEQUENCE = simtest.ctemask.config + $SIMTEST_AUTO = simtest.ctemask.auto +end + +macro simtest.setup.flatcorr + $PPSIM_RECIPE = FLATCORR + $SIMTEST_SEQUENCE = simtest.flatcorr.config + $SIMTEST_AUTO = simtest.flatcorr.auto +end + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra $CENTER_RA -dec $CENTER_DEC -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 +$BaseOptions = $BaseOptions -nx 5000 -ny 5000 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 +$RefOptions = $RefOptions -D PSF.MODEL PS_MODEL_GAUSS +$RefOptions = $RefOptions -Df STARS.SIGMA.LIM 0.5 +$RefOptions = $RefOptions -Db PSF.CONVOLVE T + +# create a reference database of fake stars to be used by ppSim below +macro mkref + if ($0 != 3) + echo "mkref (refbase) (density)" + break + end + + local refbase + $refbase = $1 + + exec rm -rf $refbase.catdir + exec rm -f $refbase.fits + + $RefOptions = $RefOptions -Df STARS.DENSITY $2 + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR $refbase.catdir + + exec ppSim $RefOptions $RefConfig $refbase + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR $refbase.catdir -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes $refbase.cmf -quick-airmass + exec relphot -averages -D CATDIR $refbase.catdir -update -region 260 280 -33 -13 +end + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.quickmos.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.quickmos.auto (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.quickmos.auto (revision 43109) @@ -0,0 +1,74 @@ + +automate MULTI + +# stages cycle between states of 'INIT.xxx' and 'RUN.xxx' + +# a stage starts in state INIT.BLOCK. + +# the "automate.block" task runs the 'block' command regularly on +# stages in state INIT.BLOCK. if it returns a valid result, the block +# is set and the stage remains in state INIT.BLOCK. if there is no +# result (ie, no matching BIAS, etc), the stage is put to state +# INIT.CHECK. + +# the "automate.check" tasks runs the 'check' command regularly on +# stages in state INIT.CHECK. if it returns 'ncheck' valid entries +# (default ncheck = 1), the stage is set to state INIT.LAUNCH. + +# the "automate.launch" tasks runs the 'launch' command once on stages +# in state INIT.LAUNCH. after the command is run, the state goes to +# RUN.LAUNCH. After the command has been successfully completed, the +# state goes to DONE.LAUNCH + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +automate METADATA + name STR BIAS + check STR "regtool -dbname @DBNAME@ -processedexp -exp_type BIAS -inst @CAMERA@" + ncheck S32 3 + launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS" + block STR "dettool -dbname @DBNAME@ -runs -active -det_type BIAS" +END + +automate METADATA + name STR DARK + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type BIAS -time 2010/01/01" + launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK" + block STR "dettool -dbname @DBNAME@ -runs -active -det_type DARK" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type DARK -time 2010/01/01" + launch STR "dettool -dbname @DBNAME@ -definebyquery -time_begin 2010/01/01 -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r" + block STR "dettool -dbname @DBNAME@ -runs -active -det_type FLAT -filter r" +END + +automate METADATA + name STR OBJECT-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -filter r -time 2010/01/01" + launch STR "chiptool -dbname @DBNAME@ -definebyquery -inst @CAMERA@ -filter r -set_label proc -set_data_group simtest_r -set_dist_group simtest -set_workdir file://@CWD@/chip" + block STR "chiptool -dbname @DBNAME@ -listrun -filter r -label proc" +END + +### Stack automation +automate METADATA + name STR STACK + check STR "warptool -dbname @DBNAME@ -listrun -filter r -label proc -state full" + ncheck S32 6 + launch STR "stacktool -dbname @DBNAME@ -definebyquery -select_filter r -set_label proc -set_data_group simtest_r -set_dist_group simtest -set_workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -min_num 4" + block STR "stacktool -dbname @DBNAME@ -listrun -filter r -label proc" +END + +### Diff automation +automate METADATA + name STR DIFF + check STR "stacktool -dbname @DBNAME@ -listrun -filter r -label proc -state full" + ncheck S32 9 + launch STR "difftool -dbname @DBNAME@ -definewarpstack -filter r -stack_label proc -warp_label proc -set_label proc -set_data_group simtest_r -set_dist_group simtest -set_workdir file://@CWD@/diff -good_frac 0.2" + block STR "difftool -dbname @DBNAME@ -listrun -filter r -label proc" +END Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.quickmos.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.quickmos.config (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.quickmos.config (revision 43109) @@ -0,0 +1,62 @@ + +CAMERA STR SIMMOSAIC + +# allows for multi-file outputs to be selected by ppSimSequence +FILERULE STR *fits + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + NIMAGES S32 3 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 2, 2 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + + FILTERS STR r + @EXPTIMES F32 30.0 + + NSETUP S32 3 +END + +SEQUENCE METADATA + OBSTYPE STR OBJECT + + CENTER.RA F32 270.0 + CENTER.DEC F32 30.0 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 2 + + DVODB STR NONE + FILTERS STR r + @EXPTIMES F32 10.00 + @SKYMAGS F32 20.86 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.9 + IQ_MAX F32 1.2 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + + MJD_MIN F32 60000.0 + MJD_MAX F32 60010.0 +END Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.simmosaic.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.simmosaic.config (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.simmosaic.config (revision 43109) @@ -0,0 +1,110 @@ + +CAMERA STR SIMMOSAIC + +FILERULE STR *.fits + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +SEQUENCE METADATA + OBSTYPE STR OBJECT + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + + +SEQUENCE METADATA + OBSTYPE STR OBJECT + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 1 + DITHER.ND S32 1 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,r,i,i + @EXPTIMES F32 5.0, 240.0, 5.0, 240.0 + @SKYMAGS F32 20.86,20.86,20.15,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.stack.auto =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.stack.auto (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.stack.auto (revision 43109) @@ -0,0 +1,65 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 20 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-i + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@" +END + +automate METADATA + name STR OBJECT-i + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" + block STR NONE +END + +automate METADATA + name STR OBJECT-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc" + block STR NONE +END + +### Stack automation??? +automate METADATA + name STR STACK + regular STR "stacktool -definebyquery -label proc -workdir file://@CWD@/stack -min_new 4 -min_num 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@" +END + Index: /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.stack.config =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.stack.config (revision 43109) +++ /branches/eam_branches/ipp-pstamp-20260421/simtest/simtest.stack.config (revision 43109) @@ -0,0 +1,76 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r + @EXPTIMES F32 0.1,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + + +SEQUENCE METADATA +### We want both short and long exposures, and dark and bright sky + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 10.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 10.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 3 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,r,r + @EXPTIMES F32 10,300,300 + @SKYMAGS F32 21,21,18 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 1.00 + IQ_MAX F32 1.00 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END +