Index: /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.sql =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.sql (revision 42598) +++ /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.sql (revision 42599) @@ -6,4 +6,20 @@ updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE skycell ( + tess_id VARCHAR(64), + skycell_id VARCHAR(64), + radeg float, + decdeg float, + glong float, + glat float, + width float, + height float, + PRIMARY KEY (tess_id, skycell_id), + KEY(radeg), + KEY(decdeg), + KEY(glong), + KEY(glat) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE TABLE Label ( @@ -15,4 +31,26 @@ KEY(priority), KEY(active) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE rawExp ( + exp_id BIGINT, + exp_name VARCHAR(64), + camera VARCHAR(64), + filter VARCHAR(64) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE chipRun ( + chip_id BIGINT, + exp_id BIGINT +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE camRun ( + cam_id BIGINT, + chip_id BIGINT +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fakeRun ( + fake_id BIGINT, + cam_id BIGINT ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -326,18 +364,62 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE skycell ( - tess_id VARCHAR(64), - skycell_id VARCHAR(64), - radeg float, - decdeg float, - glong float, - glat float, - width float, - height float, - PRIMARY KEY (tess_id, skycell_id), - KEY(radeg), - KEY(decdeg), - KEY(glong), - KEY(glat) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; - +CREATE TABLE xcForceRun ( + xcff_id BIGINT NOT NULL AUTO_INCREMENT, + xccal_id BIGINT, + sources_path_base VARCHAR(255), + state VARCHAR(64), + workdir VARCHAR(255), + label VARCHAR(64), + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + data_group VARCHAR(64), + dist_group VARCHAR(64), + note VARCHAR(255), + reduction VARCHAR(64), + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + PRIMARY KEY(xcff_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(xccal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE xcForceInput ( + xcff_id BIGINT, + warp_id BIGINT, + camera_id BIGINT, + PRIMARY KEY(xcff_id, warp_id), + FOREIGN KEY(xcff_id) REFERENCES xcForceRun(xcff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE xcForceResult ( + xcff_id BIGINT, + warp_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(16), + fault SMALLINT NOT NULL, + PRIMARY KEY(xcff_id, warp_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(xcff_id) REFERENCES xcForceRun(xcff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE xcForceSummary ( + xcff_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(16), + fault SMALLINT NOT NULL, + PRIMARY KEY(xcff_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(xcff_id) REFERENCES xcForceRun(xcff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + Index: /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh (revision 42598) +++ /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh (revision 42599) @@ -20,4 +20,6 @@ set dbuser = ipp set dbpass = ipp + +echo "******** create databases ********" # create 3 databases: two cameras and a xcamera @@ -42,7 +44,20 @@ $MYSQL -h $dbserver -u $dbuser -p$dbpass test_xcam < make.xcstack.sql +echo "******** add skycell info ********" + # generate skycell entry (needed for xcsky and xccal queries) $MYSQL -h $dbserver -u $dbuser -p$dbpass test_xcam <> test_$cam.sql - echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, path_base) values ($warp_id, '$skycell', 'Fake.Tess', '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell');" >> test_$cam.sql + echo "insert into rawExp (exp_id, exp_name, camera, filter) values ($warp_id, 'name.$warp_id', 'test_$cam', '$filter');" >> test_$cam.sql + echo "insert into chipRun (exp_id, chip_id) values ($warp_id, $warp_id);" >> test_$cam.sql + echo "insert into camRun (chip_id, cam_id) values ($warp_id, $warp_id);" >> test_$cam.sql + echo "insert into fakeRun (cam_id, fake_id) values ($warp_id, $warp_id);" >> test_$cam.sql + echo "insert into warpRun (warp_id, fake_id, tess_id, state, workdir, label) values ($warp_id, $warp_id, 'Fake.Tess', 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql + echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, quality, fault, good_frac, path_base, data_state) values ($warp_id, '$skycell', 'Fake.Tess', 0, 0, 0.8, '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell', 'full');" >> test_$cam.sql @ warp_id ++ end @@ -89,5 +110,5 @@ while ($nstack < $nwarps_max - $nwarps_min + 1) - echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql + echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$nstack.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/$skycell.$cam', 0.95, 0, 0);" >> test_$cam.sql @@ -107,4 +128,6 @@ $MYSQL -h $dbserver -u $dbuser -p$dbpass test_$cam < test_$cam.sql +echo "******** add cam2 stacks and warps ********" + # starting point for running indexes set warp_id = 1 @@ -114,4 +137,6 @@ set cam = "cam2" +rm -f test_$cam.sql + set filters = (g r g ) set skycells = (skycell.2479.024 skycell.2479.024 skycell.2479.025 ) @@ -127,6 +152,10 @@ # generate warps (two filters, two skycells) while ($warp_id <= $warp_id_max) - echo "insert into warpRun (warp_id, state, workdir, label) values ($warp_id, 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql - echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, path_base) values ($warp_id, '$skycell', 'Fake.Tess', '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell');" >> test_$cam.sql + echo "insert into rawExp (exp_id, exp_name, camera, filter) values ($warp_id, 'name.$warp_id', 'test_$cam', '$filter');" >> test_$cam.sql + echo "insert into chipRun (exp_id, chip_id) values ($warp_id, $warp_id);" >> test_$cam.sql + echo "insert into camRun (chip_id, cam_id) values ($warp_id, $warp_id);" >> test_$cam.sql + echo "insert into fakeRun (cam_id, fake_id) values ($warp_id, $warp_id);" >> test_$cam.sql + echo "insert into warpRun (warp_id, fake_id, tess_id, state, workdir, label) values ($warp_id, $warp_id, 'Fake.Tess', 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql + echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, quality, fault, good_frac, path_base, data_state) values ($warp_id, '$skycell', 'Fake.Tess', 0, 0, 0.8, '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell', 'full');" >> test_$cam.sql @ warp_id ++ end @@ -137,5 +166,5 @@ while ($nstack < $nwarps_max - $nwarps_min + 1) - echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql + echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$nstack.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/$skycell.$cam', 0.95, 0, 0);" >> test_$cam.sql @@ -155,5 +184,7 @@ $MYSQL -h $dbserver -u $dbuser -p$dbpass test_$cam < test_$cam.sql -exit 2 +#### xcstacktool tests + +echo "******** generate the camera db entries ********" # define the xcameras: @@ -165,45 +196,74 @@ xcstacktool -dbname test_xcam -addcamera -xcamera test_cam2 -# define a xcstack: -xcstacktool -dbname test_xcam -definebyquery -xcamera test_cam1 -set_workdir $mydir -set_label Test.xcstack -xcstacktool -dbname test_xcam -addbyquery -xcamera test_cam2 -select_skycell_id skycell.2479.024 -select_instack_filter g -select_xcstack_filter g - +# exit 2 + +echo "******** xcstacktool steps ********" + +set nstack = 0 +while ($nstack < $nwarps_max - $nwarps_min + 1) + # define xcstacks by matching labels: + xcstacktool -dbname test_xcam -definebyquery -xcamera test_cam1 -set_workdir $mydir -set_label Test.xcstk.$nstack -select_label label.$nstack.cam1 + + # we have to define the relationship between instack and xcstack filters, so both must be specified + xcstacktool -dbname test_xcam -addbyquery -xcamera test_cam2 -select_instack_filter g -select_xcstack_filter g -select_instack_label label.$nstack.cam2 -select_xcstack_label Test.xcstk.$nstack + xcstacktool -dbname test_xcam -addbyquery -xcamera test_cam2 -select_instack_filter r -select_xcstack_filter r -select_instack_label label.$nstack.cam2 -select_xcstack_label Test.xcstk.$nstack + + @ nstack ++ +end + +# can restrict by skycell as well +# xcstacktool -dbname test_xcam -addbyquery -xcamera test_cam2 -select_skycell_id skycell.2479.024 -select_instack_filter g -select_xcstack_filter g -select_instack_label label.0.cam2 + +# XXX shouldn't this require some restrictions? xcstacktool -dbname test_xcam -tosum > xcstack.md -set stk_id = `grep xcstack_id xcstack.md | awk '{print $3}'` -set workdir = `grep workdir xcstack.md | awk '{print $3}'` - -## run xcstack_skycell.pl or fake it -if (0) then - # manually run xcstack : note the camera must match a real camera in ippconfig/system.conf - xcstack_skycell.pl --dbname test_xcam --xcstack_id $stk_id --outroot $workdir/test.xcs.v0 --run-state new --camera GPC1 -endif - # pretend to run xcstack - xcstacktool -dbname test_xcam -addsumskyfile -path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.stk.1 -hostname elepaio -dtime_script 45.9999994374812 -xcstack_id $stk_id -endif - -# if neither of the above are set, use pantasks to run xcstack -xcskytool -dbname test_xcam -definebyquery -select_xcstack_id 1 -set_workdir $mydir/xcsky -set_label Test.xcsky - -xcskytool -dbname test_xcam -todo -label Test.xcsky - -xcskytool -dbname test_xcam -updaterun -label Test.xcsky -set_data_group foo -set_dist_group bar -set_note myNote - -xcskytool -dbname test_xcam -todo -label Test.xcsky > xcsky.md -set sky_id = `grep xcsky_id xcsky.md | awk '{print $3}'` -cat xcsky.md - -xcskytool -dbname test_xcam -inputs -xcsky_id $sky_id > xcsky.md -set workdir = `grep workdir xcsky.md | awk '{print $3}'` -cat xcsky.md - -## run xcsky.pl or fake it -if (0) then - xcsky.pl --dbname test_xcam --xcsky_id $sky_id --outroot $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.sky.1 --camera GPC1 -else - xcskytool -dbname test_xcam -addresult -path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.sky.1 -hostname foobar -dtime_script 30 -dtime_phot 28 -sources 100 -num_inputs 1 -good_frac 0.5 -xcsky_id $sky_id -endif - -xcskytool -dbname test_xcam -result -xcsky_id $sky_id - +set stk_ids = (`grep xcstack_id xcstack.md | awk '{print $3}'`) +set Tskycells = (`grep skycell_id xcstack.md | awk '{print $3}'`) +set workdir = `grep workdir xcstack.md | awk '{print $3}' | head -n 1` + +# pretend to run xcstack +set n = 1 +while ($n <= $#stk_ids) + xcstacktool -dbname test_xcam -addsumskyfile -path_base $workdir/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].stk.$stk_ids[$n] -hostname elepaio -dtime_script 45.9999994374812 -xcstack_id $stk_ids[$n] + @ n++ +end + +# exit 2 + +echo "******** xcskytool steps (static sky) ********" + +#### xcskytool tests + +# do the first set as separate filters +xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.0 -select_label Test.xcstk.0 -select_filter r +xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.0 -select_label Test.xcstk.0 -select_filter g + +# do the second and third sets as multi-filters +xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.1 -select_label Test.xcstk.1 -select_filter g -select_filter r +xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.2 -select_label Test.xcstk.2 -select_filter g -select_filter r + +xcskytool -dbname test_xcam -todo -label Test.xcsky.0 + +xcskytool -dbname test_xcam -updaterun -label Test.xcsky.0 -set_data_group foo -set_dist_group bar -set_note myNote + +## now I need to identify the entries and set fake results + +## inject results for (Test.xcsky.t0, Test.xcsky.t1, Test.xcsky.t2) + +foreach label (0 1 2) + xcskytool -dbname test_xcam -todo -label Test.xcsky.$label > xcsky.md + set sky_ids = (`grep xcsky_id xcsky.md | awk '{print $3}'`) + set Tskycells = (`grep skycell_id xcsky.md | awk '{print $3}'`) + set workdir = (`grep workdir xcsky.md | awk '{print $3}'`) + + ## pretend to run xcsky.pl + set n = 1 + while ($n <= $#sky_ids) + xcskytool -dbname test_xcam -addresult -path_base $workdir[1]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].sky.$sky_ids[$n] -hostname elepaio -dtime_script 45 -dtime_phot 28 -sources 100 -num_inputs 1 -good_frac 0.5 -xcsky_id $sky_ids[$n] + xcskytool -dbname test_xcam -result -xcsky_id $sky_ids[$n] + @ n++ + end +end + +# test the revert process: if (0) then xcskytool -dbname test_xcam -updateresult -xcsky_id 1 -fault 1 @@ -212,21 +272,36 @@ endif +####### xccal + +echo "******** xcskytool steps (calibration) ********" + +# queue a single xcskyRun to xccalRun: +# xcskytool -dbname test_xcam -definexccalrun -select_xcsky_id 1 + +# queue all three labels +foreach label (0 1 2) + xcskytool -dbname test_xcam -definexccalrun -select_label Test.xcsky.$label -set_label Test.xccal.$label +end + +# mysql> select xccal_id, xcsky_id, xcstack_id, xccalRun.label as calLabel, xcskyRun.label as skyLabel, xcstackRun.label as stkLabel, xcstackRun.filter, xcskyResult.path_base, xcstackSumSkyfile.path_base from xccalRun join xcskyInput using (xcsky_id, xcstack_id) join xcskyRun using (xcsky_id) join xcskyResult using (xcsky_id) join xcstackRun using (xcstack_id) join xcstackSumSkyfile using (xcstack_id); + # NOTE: xccalRun inherits the label, data_group, dist_group, reduction from xcskyRun -xcskytool -dbname test_xcam -definexccalrun -select_xcsky_id 1 - -xcskytool -dbname test_xcam -pendingxccalrun -label Test.xcsky > xccal.md -set cal_id = `grep xccal_id xccal.md | awk '{print $3}'` -cat xccal.md - -## run xccal.pl or fake it -if (0) then - echo xccal.pl --xccal_id 1 --xcstack_id 1 --outroot $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.cal.1 --path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.sky.1 --camera GPC1 --dbname test_xcam - xccal.pl --xccal_id 1 --xcstack_id 1 --outroot $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.cal.1 --path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.sky.1 --camera GPC1 --dbname test_xcam -else - xcskytool -dbname test_xcam -addxccalresult -xccal_id 1 -path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.cal.1 -hostname foobaz -zpt_obs 25.0 -endif - -xcskytool -dbname test_xcam -xccalresult -xccal_id $cal_id - + +foreach label (0 1 2) + xcskytool -dbname test_xcam -pendingxccalrun -label Test.xccal.$label > xccal.md + set cal_ids = (`grep xccal_id xccal.md | awk '{print $3}'`) + set Tskycells = (`grep skycell_id xccal.md | awk '{print $3}'`) + set workdir = (`grep workdir xccal.md | awk '{print $3}'`) + + ## pretend to run xcsky.pl + set n = 1 + while ($n <= $#cal_ids) + xcskytool -dbname test_xcam -addxccalresult -path_base $workdir[1]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].cal.$cal_ids[$n] -hostname elepaio -zpt_obs 25.0 -xccal_id $cal_ids[$n] + xcskytool -dbname test_xcam -xccalresult -xccal_id $cal_ids[$n] + @ n++ + end +end + +# test the revert process if (0) then xcskytool -dbname test_xcam -updatexccal -xccal_id 1 -set_fault 1 @@ -235,2 +310,56 @@ endif +####### xcfftool + +echo "******** xcfftool steps ********" + +foreach label (0 1 2) + xcfftool -dbname test_cam1 -definebyquery -select_xccal_label Test.xccal.$label -set_label Test.xcff.$label -set_workdir here/xcff -xcamera test_xcam + xcfftool -dbname test_cam2 -definebyquery -select_xccal_label Test.xccal.$label -set_label Test.xcff.$label -set_workdir here/xcff -xcamera test_xcam +end + +xcfftool -dbname test_cam1 -updaterun -xcff_id 1 -set_note myNote + +foreach label (0 1 2) + set cam = cam1 + xcfftool -dbname test_$cam -todo -label Test.xcff.$label > xcff.todo.md + set xcff_ids = (`grep xcff_id xcff.todo.md | awk '{print $3}'`) + set warp_ids = (`grep warp_id xcff.todo.md | awk '{print $3}'`) + + ## pretend to run xcsky.pl + set n = 1 + while ($n <= $#xcff_ids) + xcfftool -dbname test_$cam -addresult -xcff_id $xcff_ids[$n] -warp_id $warp_ids[$n] -path_base path/Fake.Tess.xcff.$xcff_ids[$n] -hostname elepaio -quality 0 -fault 0 + xcfftool -dbname test_$cam -result -xcff_id $xcff_ids[$n] + @ n++ + end + + # get a list of the unique xcff_ids + set xcff_ids = (`grep xcff_id xcff.todo.md | awk '{print $3}' | sort | uniq`) + + ## pretend to run xcsky.pl + set n = 1 + while ($n <= $#xcff_ids) + xcfftool -dbname test_$cam -toadvance -xcff_id $xcff_ids[$n] + xcfftool -dbname test_$cam -addsummary -xcff_id $xcff_ids[$n] -path_base path/Fake.Tess.xcff.$xcff_ids[$n] -dtime_script 0.0 -quality 0 -fault 0 -hostname elepaio + xcfftool -dbname test_$cam -summary -xcff_id $xcff_ids[$n] + @ n++ + end +end + +# test the revert process +if (0) then + xcfftool -dbname test_$cam -updateresult -xcff_id 1 -warp_id 6 -fault 1 + xcfftool -dbname test_$cam -updateresult -xcff_id 1 -warp_id 7 -fault 1 + xcfftool -dbname test_$cam -updateresult -xcff_id 1 -warp_id 8 -fault 1 + xcfftool -dbname test_$cam -revert -xcff_id 1 -warp_id 6 + xcfftool -dbname test_$cam -revert -xcff_id 1 -warp_id 7 + xcfftool -dbname test_$cam -revert -xcff_id 1 -warp_id 8 + xcfftool -dbname test_$cam -updatesummary -xcff_id 1 -fault 1 + xcfftool -dbname test_$cam -revertsummary -xccal_id 1 +endif + +# -exportrun list summary results +# -importrun list summary results + +exit 2