Changeset 42599 for branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh
- Timestamp:
- Jan 20, 2024, 10:45:45 AM (2 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh
r42594 r42599 20 20 set dbuser = ipp 21 21 set dbpass = ipp 22 23 echo "******** create databases ********" 22 24 23 25 # create 3 databases: two cameras and a xcamera … … 42 44 $MYSQL -h $dbserver -u $dbuser -p$dbpass test_xcam < make.xcstack.sql 43 45 46 echo "******** add skycell info ********" 47 44 48 # generate skycell entry (needed for xcsky and xccal queries) 45 49 $MYSQL -h $dbserver -u $dbuser -p$dbpass test_xcam <<EOF 46 insert into skycell (tess_id, skycell_id, radeg, decdeg, glong, glat, width, height) values ('Fake.Tess', 'skycell.2479.024', 200.412, 60.9857, 116.426, 55.7593, 0.433333, 0.438819); 50 insert into skycell (tess_id, skycell_id, radeg, decdeg, width, height) values ('Fake.Tess', 'skycell.2479.024', 200.4, 60.0, 0.433333, 0.438819); 51 insert into skycell (tess_id, skycell_id, radeg, decdeg, width, height) values ('Fake.Tess', 'skycell.2479.025', 200.5, 60.0, 0.433333, 0.438819); 52 insert into dbversion (schema_version) values ('0.1'); 53 EOF 54 $MYSQL -h $dbserver -u $dbuser -p$dbpass test_cam1 <<EOF 55 insert into skycell (tess_id, skycell_id, radeg, decdeg, width, height) values ('Fake.Tess', 'skycell.2479.024', 200.4, 60.0, 0.433333, 0.438819); 56 insert into skycell (tess_id, skycell_id, radeg, decdeg, width, height) values ('Fake.Tess', 'skycell.2479.025', 200.5, 60.0, 0.433333, 0.438819); 57 insert into dbversion (schema_version) values ('0.1'); 58 EOF 59 $MYSQL -h $dbserver -u $dbuser -p$dbpass test_cam2 <<EOF 60 insert into skycell (tess_id, skycell_id, radeg, decdeg, width, height) values ('Fake.Tess', 'skycell.2479.024', 200.4, 60.0, 0.433333, 0.438819); 61 insert into skycell (tess_id, skycell_id, radeg, decdeg, width, height) values ('Fake.Tess', 'skycell.2479.025', 200.5, 60.0, 0.433333, 0.438819); 47 62 insert into dbversion (schema_version) values ('0.1'); 48 63 EOF … … 57 72 # for each stack (to mimic, e.g., a new stack made with additional new warps) 58 73 59 rm -f test_cam1.sql 74 echo "******** add cam1 stacks and warps ********" 60 75 61 76 # starting point for running indexes … … 66 81 set cam = "cam1" 67 82 83 rm -f test_$cam.sql 84 68 85 set filters = (g r g ) 69 86 set skycells = (skycell.2479.024 skycell.2479.024 skycell.2479.025 ) … … 79 96 # generate warps (two filters, two skycells) 80 97 while ($warp_id <= $warp_id_max) 81 echo "insert into warpRun (warp_id, state, workdir, label) values ($warp_id, 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql 82 echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, path_base) values ($warp_id, '$skycell', 'Fake.Tess', '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell');" >> test_$cam.sql 98 echo "insert into rawExp (exp_id, exp_name, camera, filter) values ($warp_id, 'name.$warp_id', 'test_$cam', '$filter');" >> test_$cam.sql 99 echo "insert into chipRun (exp_id, chip_id) values ($warp_id, $warp_id);" >> test_$cam.sql 100 echo "insert into camRun (chip_id, cam_id) values ($warp_id, $warp_id);" >> test_$cam.sql 101 echo "insert into fakeRun (cam_id, fake_id) values ($warp_id, $warp_id);" >> test_$cam.sql 102 echo "insert into warpRun (warp_id, fake_id, tess_id, state, workdir, label) values ($warp_id, $warp_id, 'Fake.Tess', 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql 103 echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, quality, fault, good_frac, path_base, data_state) values ($warp_id, '$skycell', 'Fake.Tess', 0, 0, 0.8, '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell', 'full');" >> test_$cam.sql 83 104 @ warp_id ++ 84 105 end … … 89 110 while ($nstack < $nwarps_max - $nwarps_min + 1) 90 111 91 echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$ cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql112 echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$nstack.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql 92 113 echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/$skycell.$cam', 0.95, 0, 0);" >> test_$cam.sql 93 114 … … 107 128 $MYSQL -h $dbserver -u $dbuser -p$dbpass test_$cam < test_$cam.sql 108 129 130 echo "******** add cam2 stacks and warps ********" 131 109 132 # starting point for running indexes 110 133 set warp_id = 1 … … 114 137 set cam = "cam2" 115 138 139 rm -f test_$cam.sql 140 116 141 set filters = (g r g ) 117 142 set skycells = (skycell.2479.024 skycell.2479.024 skycell.2479.025 ) … … 127 152 # generate warps (two filters, two skycells) 128 153 while ($warp_id <= $warp_id_max) 129 echo "insert into warpRun (warp_id, state, workdir, label) values ($warp_id, 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql 130 echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, path_base) values ($warp_id, '$skycell', 'Fake.Tess', '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell');" >> test_$cam.sql 154 echo "insert into rawExp (exp_id, exp_name, camera, filter) values ($warp_id, 'name.$warp_id', 'test_$cam', '$filter');" >> test_$cam.sql 155 echo "insert into chipRun (exp_id, chip_id) values ($warp_id, $warp_id);" >> test_$cam.sql 156 echo "insert into camRun (chip_id, cam_id) values ($warp_id, $warp_id);" >> test_$cam.sql 157 echo "insert into fakeRun (cam_id, fake_id) values ($warp_id, $warp_id);" >> test_$cam.sql 158 echo "insert into warpRun (warp_id, fake_id, tess_id, state, workdir, label) values ($warp_id, $warp_id, 'Fake.Tess', 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql 159 echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, quality, fault, good_frac, path_base, data_state) values ($warp_id, '$skycell', 'Fake.Tess', 0, 0, 0.8, '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell', 'full');" >> test_$cam.sql 131 160 @ warp_id ++ 132 161 end … … 137 166 while ($nstack < $nwarps_max - $nwarps_min + 1) 138 167 139 echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$ cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql168 echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$nstack.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql 140 169 echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/$skycell.$cam', 0.95, 0, 0);" >> test_$cam.sql 141 170 … … 155 184 $MYSQL -h $dbserver -u $dbuser -p$dbpass test_$cam < test_$cam.sql 156 185 157 exit 2 186 #### xcstacktool tests 187 188 echo "******** generate the camera db entries ********" 158 189 159 190 # define the xcameras: … … 165 196 xcstacktool -dbname test_xcam -addcamera -xcamera test_cam2 166 197 167 # define a xcstack: 168 xcstacktool -dbname test_xcam -definebyquery -xcamera test_cam1 -set_workdir $mydir -set_label Test.xcstack 169 xcstacktool -dbname test_xcam -addbyquery -xcamera test_cam2 -select_skycell_id skycell.2479.024 -select_instack_filter g -select_xcstack_filter g 170 198 # exit 2 199 200 echo "******** xcstacktool steps ********" 201 202 set nstack = 0 203 while ($nstack < $nwarps_max - $nwarps_min + 1) 204 # define xcstacks by matching labels: 205 xcstacktool -dbname test_xcam -definebyquery -xcamera test_cam1 -set_workdir $mydir -set_label Test.xcstk.$nstack -select_label label.$nstack.cam1 206 207 # we have to define the relationship between instack and xcstack filters, so both must be specified 208 xcstacktool -dbname test_xcam -addbyquery -xcamera test_cam2 -select_instack_filter g -select_xcstack_filter g -select_instack_label label.$nstack.cam2 -select_xcstack_label Test.xcstk.$nstack 209 xcstacktool -dbname test_xcam -addbyquery -xcamera test_cam2 -select_instack_filter r -select_xcstack_filter r -select_instack_label label.$nstack.cam2 -select_xcstack_label Test.xcstk.$nstack 210 211 @ nstack ++ 212 end 213 214 # can restrict by skycell as well 215 # xcstacktool -dbname test_xcam -addbyquery -xcamera test_cam2 -select_skycell_id skycell.2479.024 -select_instack_filter g -select_xcstack_filter g -select_instack_label label.0.cam2 216 217 # XXX shouldn't this require some restrictions? 171 218 xcstacktool -dbname test_xcam -tosum > xcstack.md 172 set stk_id = `grep xcstack_id xcstack.md | awk '{print $3}'` 173 set workdir = `grep workdir xcstack.md | awk '{print $3}'` 174 175 ## run xcstack_skycell.pl or fake it 176 if (0) then 177 # manually run xcstack : note the camera must match a real camera in ippconfig/system.conf 178 xcstack_skycell.pl --dbname test_xcam --xcstack_id $stk_id --outroot $workdir/test.xcs.v0 --run-state new --camera GPC1 179 endif 180 # pretend to run xcstack 181 xcstacktool -dbname test_xcam -addsumskyfile -path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.stk.1 -hostname elepaio -dtime_script 45.9999994374812 -xcstack_id $stk_id 182 endif 183 184 # if neither of the above are set, use pantasks to run xcstack 185 xcskytool -dbname test_xcam -definebyquery -select_xcstack_id 1 -set_workdir $mydir/xcsky -set_label Test.xcsky 186 187 xcskytool -dbname test_xcam -todo -label Test.xcsky 188 189 xcskytool -dbname test_xcam -updaterun -label Test.xcsky -set_data_group foo -set_dist_group bar -set_note myNote 190 191 xcskytool -dbname test_xcam -todo -label Test.xcsky > xcsky.md 192 set sky_id = `grep xcsky_id xcsky.md | awk '{print $3}'` 193 cat xcsky.md 194 195 xcskytool -dbname test_xcam -inputs -xcsky_id $sky_id > xcsky.md 196 set workdir = `grep workdir xcsky.md | awk '{print $3}'` 197 cat xcsky.md 198 199 ## run xcsky.pl or fake it 200 if (0) then 201 xcsky.pl --dbname test_xcam --xcsky_id $sky_id --outroot $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.sky.1 --camera GPC1 202 else 203 xcskytool -dbname test_xcam -addresult -path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.sky.1 -hostname foobar -dtime_script 30 -dtime_phot 28 -sources 100 -num_inputs 1 -good_frac 0.5 -xcsky_id $sky_id 204 endif 205 206 xcskytool -dbname test_xcam -result -xcsky_id $sky_id 207 219 set stk_ids = (`grep xcstack_id xcstack.md | awk '{print $3}'`) 220 set Tskycells = (`grep skycell_id xcstack.md | awk '{print $3}'`) 221 set workdir = `grep workdir xcstack.md | awk '{print $3}' | head -n 1` 222 223 # pretend to run xcstack 224 set n = 1 225 while ($n <= $#stk_ids) 226 xcstacktool -dbname test_xcam -addsumskyfile -path_base $workdir/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].stk.$stk_ids[$n] -hostname elepaio -dtime_script 45.9999994374812 -xcstack_id $stk_ids[$n] 227 @ n++ 228 end 229 230 # exit 2 231 232 echo "******** xcskytool steps (static sky) ********" 233 234 #### xcskytool tests 235 236 # do the first set as separate filters 237 xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.0 -select_label Test.xcstk.0 -select_filter r 238 xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.0 -select_label Test.xcstk.0 -select_filter g 239 240 # do the second and third sets as multi-filters 241 xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.1 -select_label Test.xcstk.1 -select_filter g -select_filter r 242 xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.2 -select_label Test.xcstk.2 -select_filter g -select_filter r 243 244 xcskytool -dbname test_xcam -todo -label Test.xcsky.0 245 246 xcskytool -dbname test_xcam -updaterun -label Test.xcsky.0 -set_data_group foo -set_dist_group bar -set_note myNote 247 248 ## now I need to identify the entries and set fake results 249 250 ## inject results for (Test.xcsky.t0, Test.xcsky.t1, Test.xcsky.t2) 251 252 foreach label (0 1 2) 253 xcskytool -dbname test_xcam -todo -label Test.xcsky.$label > xcsky.md 254 set sky_ids = (`grep xcsky_id xcsky.md | awk '{print $3}'`) 255 set Tskycells = (`grep skycell_id xcsky.md | awk '{print $3}'`) 256 set workdir = (`grep workdir xcsky.md | awk '{print $3}'`) 257 258 ## pretend to run xcsky.pl 259 set n = 1 260 while ($n <= $#sky_ids) 261 xcskytool -dbname test_xcam -addresult -path_base $workdir[1]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].sky.$sky_ids[$n] -hostname elepaio -dtime_script 45 -dtime_phot 28 -sources 100 -num_inputs 1 -good_frac 0.5 -xcsky_id $sky_ids[$n] 262 xcskytool -dbname test_xcam -result -xcsky_id $sky_ids[$n] 263 @ n++ 264 end 265 end 266 267 # test the revert process: 208 268 if (0) then 209 269 xcskytool -dbname test_xcam -updateresult -xcsky_id 1 -fault 1 … … 212 272 endif 213 273 274 ####### xccal 275 276 echo "******** xcskytool steps (calibration) ********" 277 278 # queue a single xcskyRun to xccalRun: 279 # xcskytool -dbname test_xcam -definexccalrun -select_xcsky_id 1 280 281 # queue all three labels 282 foreach label (0 1 2) 283 xcskytool -dbname test_xcam -definexccalrun -select_label Test.xcsky.$label -set_label Test.xccal.$label 284 end 285 286 # mysql> select xccal_id, xcsky_id, xcstack_id, xccalRun.label as calLabel, xcskyRun.label as skyLabel, xcstackRun.label as stkLabel, xcstackRun.filter, xcskyResult.path_base, xcstackSumSkyfile.path_base from xccalRun join xcskyInput using (xcsky_id, xcstack_id) join xcskyRun using (xcsky_id) join xcskyResult using (xcsky_id) join xcstackRun using (xcstack_id) join xcstackSumSkyfile using (xcstack_id); 287 214 288 # NOTE: xccalRun inherits the label, data_group, dist_group, reduction from xcskyRun 215 xcskytool -dbname test_xcam -definexccalrun -select_xcsky_id 1 216 217 xcskytool -dbname test_xcam -pendingxccalrun -label Test.xcsky > xccal.md 218 set cal_id = `grep xccal_id xccal.md | awk '{print $3}'` 219 cat xccal.md 220 221 ## run xccal.pl or fake it 222 if (0) then 223 echo xccal.pl --xccal_id 1 --xcstack_id 1 --outroot $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.cal.1 --path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.sky.1 --camera GPC1 --dbname test_xcam 224 xccal.pl --xccal_id 1 --xcstack_id 1 --outroot $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.cal.1 --path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.sky.1 --camera GPC1 --dbname test_xcam 225 else 226 xcskytool -dbname test_xcam -addxccalresult -xccal_id 1 -path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.cal.1 -hostname foobaz -zpt_obs 25.0 227 endif 228 229 xcskytool -dbname test_xcam -xccalresult -xccal_id $cal_id 230 289 290 foreach label (0 1 2) 291 xcskytool -dbname test_xcam -pendingxccalrun -label Test.xccal.$label > xccal.md 292 set cal_ids = (`grep xccal_id xccal.md | awk '{print $3}'`) 293 set Tskycells = (`grep skycell_id xccal.md | awk '{print $3}'`) 294 set workdir = (`grep workdir xccal.md | awk '{print $3}'`) 295 296 ## pretend to run xcsky.pl 297 set n = 1 298 while ($n <= $#cal_ids) 299 xcskytool -dbname test_xcam -addxccalresult -path_base $workdir[1]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].cal.$cal_ids[$n] -hostname elepaio -zpt_obs 25.0 -xccal_id $cal_ids[$n] 300 xcskytool -dbname test_xcam -xccalresult -xccal_id $cal_ids[$n] 301 @ n++ 302 end 303 end 304 305 # test the revert process 231 306 if (0) then 232 307 xcskytool -dbname test_xcam -updatexccal -xccal_id 1 -set_fault 1 … … 235 310 endif 236 311 312 ####### xcfftool 313 314 echo "******** xcfftool steps ********" 315 316 foreach label (0 1 2) 317 xcfftool -dbname test_cam1 -definebyquery -select_xccal_label Test.xccal.$label -set_label Test.xcff.$label -set_workdir here/xcff -xcamera test_xcam 318 xcfftool -dbname test_cam2 -definebyquery -select_xccal_label Test.xccal.$label -set_label Test.xcff.$label -set_workdir here/xcff -xcamera test_xcam 319 end 320 321 xcfftool -dbname test_cam1 -updaterun -xcff_id 1 -set_note myNote 322 323 foreach label (0 1 2) 324 set cam = cam1 325 xcfftool -dbname test_$cam -todo -label Test.xcff.$label > xcff.todo.md 326 set xcff_ids = (`grep xcff_id xcff.todo.md | awk '{print $3}'`) 327 set warp_ids = (`grep warp_id xcff.todo.md | awk '{print $3}'`) 328 329 ## pretend to run xcsky.pl 330 set n = 1 331 while ($n <= $#xcff_ids) 332 xcfftool -dbname test_$cam -addresult -xcff_id $xcff_ids[$n] -warp_id $warp_ids[$n] -path_base path/Fake.Tess.xcff.$xcff_ids[$n] -hostname elepaio -quality 0 -fault 0 333 xcfftool -dbname test_$cam -result -xcff_id $xcff_ids[$n] 334 @ n++ 335 end 336 337 # get a list of the unique xcff_ids 338 set xcff_ids = (`grep xcff_id xcff.todo.md | awk '{print $3}' | sort | uniq`) 339 340 ## pretend to run xcsky.pl 341 set n = 1 342 while ($n <= $#xcff_ids) 343 xcfftool -dbname test_$cam -toadvance -xcff_id $xcff_ids[$n] 344 xcfftool -dbname test_$cam -addsummary -xcff_id $xcff_ids[$n] -path_base path/Fake.Tess.xcff.$xcff_ids[$n] -dtime_script 0.0 -quality 0 -fault 0 -hostname elepaio 345 xcfftool -dbname test_$cam -summary -xcff_id $xcff_ids[$n] 346 @ n++ 347 end 348 end 349 350 # test the revert process 351 if (0) then 352 xcfftool -dbname test_$cam -updateresult -xcff_id 1 -warp_id 6 -fault 1 353 xcfftool -dbname test_$cam -updateresult -xcff_id 1 -warp_id 7 -fault 1 354 xcfftool -dbname test_$cam -updateresult -xcff_id 1 -warp_id 8 -fault 1 355 xcfftool -dbname test_$cam -revert -xcff_id 1 -warp_id 6 356 xcfftool -dbname test_$cam -revert -xcff_id 1 -warp_id 7 357 xcfftool -dbname test_$cam -revert -xcff_id 1 -warp_id 8 358 xcfftool -dbname test_$cam -updatesummary -xcff_id 1 -fault 1 359 xcfftool -dbname test_$cam -revertsummary -xccal_id 1 360 endif 361 362 # -exportrun list summary results 363 # -importrun list summary results 364 365 exit 2
Note:
See TracChangeset
for help on using the changeset viewer.
