Index: /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/get.schema.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/get.schema.sh (revision 42611) +++ /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/get.schema.sh (revision 42611) @@ -0,0 +1,52 @@ +#!/bin/csh + +if ($#argv != 1) then + echo "USAGE: get.schema.sh (database)" + exit 2 +endif + +which mariadb >& /dev/null +@ has_mariadb = ($status == 0) + +which mysql >& /dev/null +@ has_mysql = ($status == 0) + +set MYSQL = none +if ($has_mariadb) then + set MYSQL = mariadb +endif +if (($has_mariadb == 0) && ($has_mysql)) then + set MYSQL = mysql +endif + +echo "mysql is $MYSQL" + +set dbserver = localhost +set dbuser = ipp +set dbpass = ipp +set dbname = $argv[1] + +# generate skycell entry (needed for xcsky and xccal queries) +$MYSQL -h $dbserver -u $dbuser -p$dbpass $dbname < tables.$dbname.txt + show tables +EOF +if ($status) then + echo "ERROR: failed in mysql load for $dbname" + exit 2 + endif + +rm -f tables.$dbname.sql +foreach f (`cat tables.$dbname.txt | awk '(NR > 1){print $1}'`) + echo "SHOW CREATE TABLE $f;" >> tables.$dbname.sql + echo -n "$f, " +end +echo "" + +# generate skycell entry (needed for xcsky and xccal queries) +$MYSQL -N -h $dbserver -u $dbuser -p$dbpass $dbname < tables.$dbname.sql > create.$dbname.txt +if ($status) then + echo "ERROR: failed in mysql load for $dbname" + exit 2 + endif + + Index: /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh (revision 42610) +++ /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh (revision 42611) @@ -40,7 +40,13 @@ EOF -$MYSQL -h $dbserver -u $dbuser -p$dbpass test_cam1 < make.xcstack.sql -$MYSQL -h $dbserver -u $dbuser -p$dbpass test_cam2 < make.xcstack.sql -$MYSQL -h $dbserver -u $dbuser -p$dbpass test_xcam < make.xcstack.sql +pxadmin -dbname test_cam1 -create +pxadmin -dbname test_cam2 -create +pxadmin -dbname test_xcam -create + +exit 2 + +#$MYSQL -h $dbserver -u $dbuser -p$dbpass test_cam1 < make.xcstack.sql +#$MYSQL -h $dbserver -u $dbuser -p$dbpass test_cam2 < make.xcstack.sql +#$MYSQL -h $dbserver -u $dbuser -p$dbpass test_xcam < make.xcstack.sql echo "******** add skycell info ********" @@ -52,4 +58,9 @@ insert into dbversion (schema_version) values ('0.1'); EOF +if ($status) then + echo "ERROR: failed in mysql load for test_xcam" + exit 2 + endif + $MYSQL -h $dbserver -u $dbuser -p$dbpass test_cam1 < test_$cam.sql set filters = (g r g ) @@ -96,5 +117,5 @@ # generate warps (two filters, two skycells) while ($warp_id <= $warp_id_max) - echo "insert into rawExp (exp_id, exp_name, camera, filter) values ($warp_id, 'name.$warp_id', 'test_$cam', '$filter');" >> test_$cam.sql + echo "insert into rawExp (exp_id, exp_name, camera, filter, fault) values ($warp_id, 'name.$warp_id', 'test_$cam', '$filter', 0);" >> test_$cam.sql echo "insert into chipRun (exp_id, chip_id) values ($warp_id, $warp_id);" >> test_$cam.sql echo "insert into camRun (chip_id, cam_id) values ($warp_id, $warp_id);" >> test_$cam.sql @@ -127,4 +148,8 @@ $MYSQL -h $dbserver -u $dbuser -p$dbpass test_$cam < test_$cam.sql +if ($status) then + echo "ERROR: failed in mysql inserts for test_cam1" + exit 2 + endif echo "******** add cam2 stacks and warps ********" @@ -138,4 +163,5 @@ rm -f test_$cam.sql +echo "SET FOREIGN_KEY_CHECKS=0;" > test_$cam.sql set filters = (g r g ) @@ -152,5 +178,5 @@ # generate warps (two filters, two skycells) while ($warp_id <= $warp_id_max) - echo "insert into rawExp (exp_id, exp_name, camera, filter) values ($warp_id, 'name.$warp_id', 'test_$cam', '$filter');" >> test_$cam.sql + echo "insert into rawExp (exp_id, exp_name, camera, filter, fault) values ($warp_id, 'name.$warp_id', 'test_$cam', '$filter', 0);" >> test_$cam.sql echo "insert into chipRun (exp_id, chip_id) values ($warp_id, $warp_id);" >> test_$cam.sql echo "insert into camRun (chip_id, cam_id) values ($warp_id, $warp_id);" >> test_$cam.sql @@ -222,5 +248,5 @@ mkdir -p $workdir -# pretend to run xcstack +# (pretend to) run xcstack set n = 1 while ($n <= $#stk_ids) @@ -263,5 +289,5 @@ set workdir = (`grep workdir xcsky.md | awk '{print $3}'`) - ## pretend to run xcsky.pl + ## (pretend to) run xcsky.pl set n = 1 while ($n <= $#sky_ids) @@ -310,8 +336,9 @@ set pathbase = (`grep path_base xccal.md | awk '{print $3}'`) - ## pretend to run xcsky.pl + ## (pretend to) run xccal.pl set n = 1 while ($n <= $#cal_ids) # OUTROOT is defined in xccal.pro based on workdir : "%s/%s/%s/%s.%s.cal.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $SKYCAL_ID ****** TBD ***** + # change 'cal' to 'xccal' here? if (0 && ($n == 1)) then mkdir -p $workdir[$n]/Fake.Tess/$Tskycells[$n] @@ -344,80 +371,87 @@ xcfftool -dbname test_cam1 -updaterun -xcff_id 1 -set_note myNote -foreach label (0 1 2) - set cam = cam1 - xcfftool -dbname test_$cam -todo -label Test.xcff.$label > xcff.todo.md - set xcff_ids = (`grep xcff_id xcff.todo.md | awk '{print $3}'`) - set warp_ids = (`grep warp_id xcff.todo.md | awk '{print $3}'`) - set Tskycells = (`grep skycell_id xcff.todo.md | awk '{print $3}'`) - set workdir = (`grep workdir xcff.todo.md | awk '{print $3}'`) - set warppaths = (`grep warp_path_base xcff.todo.md | awk '{print $3}'`) - set srcpaths = (`grep sources_path_base xcff.todo.md | awk '{print $3}'`) - - ## pretend to run xcsky.pl - set n = 1 - while ($n <= $#xcff_ids) - # OUTROOT is defined in xcff.pro based on workdir : "%s/%s/%s/%s.%s.wrp.%s.xcff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $WARP_ID $FF_ID ***** TBD ***** - if (1 && ($cam == "cam1") && ($xcff_ids[$n] == 2)) then - mkdir -p $workdir[$n]/Fake.Tess/$Tskycells[$n] - echo xcff_warp.pl --dbname test_$cam --xcff_id $xcff_ids[$n] --warp_id $warp_ids[$n] --skycell_id $Tskycells[$n] --warp_path_base $warppaths[$n] --sources_path_base $srcpaths[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].wrp.$warp_ids[$n].xcff.$xcff_ids[$n] --camera GPC1 - xcff_warp.pl --dbname test_$cam --xcff_id $xcff_ids[$n] --warp_id $warp_ids[$n] --skycell_id $Tskycells[$n] --warp_path_base $warppaths[$n] --sources_path_base $srcpaths[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].wrp.$warp_ids[$n].xcff.$xcff_ids[$n] --camera GPC1 - else - xcfftool -dbname test_$cam -addresult -xcff_id $xcff_ids[$n] -warp_id $warp_ids[$n] -path_base $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].wrp.$warp_ids[$n].xcff.$xcff_ids[$n] -hostname elepaio -quality 0 -fault 0 - xcfftool -dbname test_$cam -result -xcff_id $xcff_ids[$n] - endif - @ n++ - end - - # get a list of the unique xcff_ids - set xcff_ids_u = (`grep xcff_id xcff.todo.md | awk '{print $3}' | sort | uniq`) - - ## pretend to run xcsky.pl - set n = 1 - while ($n <= $#xcff_ids_u) - # get the workdir from above which matches this uniq xcff_ids: - set workdir_u = "" - set skycell_u = "" - set t = 1 - while ($t <= $#xcff_ids) - if ($xcff_ids[$t] == $xcff_ids_u[$n]) then - set workdir_u = $workdir[$n] - set skycell_u = $Tskycells[$n] +foreach cam (cam1 cam2) + foreach label (0 1 2) + xcfftool -dbname test_$cam -todo -label Test.xcff.$label > xcff.todo.md + set xcff_ids = (`grep xcff_id xcff.todo.md | awk '{print $3}'`) + set xccal_ids = (`grep xccal_id xcff.todo.md | awk '{print $3}'`) + set warp_ids = (`grep warp_id xcff.todo.md | awk '{print $3}'`) + set Tskycells = (`grep skycell_id xcff.todo.md | awk '{print $3}'`) + set workdir = (`grep workdir xcff.todo.md | awk '{print $3}'`) + set warppaths = (`grep warp_path_base xcff.todo.md | awk '{print $3}'`) + set srcpaths = (`grep sources_path_base xcff.todo.md | awk '{print $3}'`) + + ## (pretend to) run xcff_warp.pl + set n = 1 + while ($n <= $#xcff_ids) + # OUTROOT is defined in xcff.pro based on workdir : "%s/%s/%s/%s.%s.wrp.%s.xcff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $WARP_ID $FF_ID ***** TBD ***** + # WORKDIR is needed to prevent collisions between (e.g.,) gpc1 and gpc2 jobs for the same skycell, or else add the camera to the output path + if (0 && ($xccal_ids[$n] == 1)) then + mkdir -p $workdir[$n]/Fake.Tess/$Tskycells[$n] + echo xcff_warp.pl --dbname test_$cam --xcff_id $xcff_ids[$n] --warp_id $warp_ids[$n] --skycell_id $Tskycells[$n] --warp_path_base $warppaths[$n] --sources_path_base $srcpaths[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].$cam.wrp.$warp_ids[$n].xcff.$xcff_ids[$n] --camera GPC1 + xcff_warp.pl --dbname test_$cam --xcff_id $xcff_ids[$n] --warp_id $warp_ids[$n] --skycell_id $Tskycells[$n] --warp_path_base $warppaths[$n] --sources_path_base $srcpaths[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].$cam.wrp.$warp_ids[$n].xcff.$xcff_ids[$n] --camera GPC1 + else + xcfftool -dbname test_$cam -addresult -xcff_id $xcff_ids[$n] -warp_id $warp_ids[$n] -path_base $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].$cam.wrp.$warp_ids[$n].xcff.$xcff_ids[$n] -hostname elepaio -quality 0 -fault 0 + xcfftool -dbname test_$cam -result -xcff_id $xcff_ids[$n] endif - @ t++ + @ n++ end - if ($workdir_u == "") then - echo "ERROR: failed to find workdir for $xcff_ids_u[$n] from $workdir" - exit 2 - endif - - # OUTROOT is defined in xcff.pro based on workdir : "%s/%s/%s/%s.%s.xcff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $XCFF_ID **** TBD *** - xcfftool -dbname test_$cam -toadvance -xcff_id $xcff_ids_u[$n] - - if (1 && ($cam == "cam1") && ($xcff_ids[$n] == 2)) then - mkdir -p $workdir[$n]/Fake.Tess/$Tskycells[$n] - echo xcff_summary.pl --dbname test_$cam --xcff_id $xcff_ids[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].xcff.$xcff_ids[$n] --camera GPC1 - xcff_summary.pl --dbname test_$cam --xcff_id $xcff_ids[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].xcff.$xcff_ids[$n] --camera GPC1 - else - xcfftool -dbname test_$cam -addsummary -xcff_id $xcff_ids_u[$n] -path_base $workdir_u/Fake.Tess/$skycell_u/Fake.Tess.$skycell_u.xcff.$xcff_ids_u[$n] -dtime_script 0.0 -quality 0 -fault 0 -hostname elepaio - endif - xcfftool -dbname test_$cam -summary -xcff_id $xcff_ids_u[$n] - @ n++ - end -end - -# test the revert process -if (0) then - xcfftool -dbname test_$cam -updateresult -xcff_id 1 -warp_id 6 -fault 1 - xcfftool -dbname test_$cam -updateresult -xcff_id 1 -warp_id 7 -fault 1 - xcfftool -dbname test_$cam -updateresult -xcff_id 1 -warp_id 8 -fault 1 - xcfftool -dbname test_$cam -revert -xcff_id 1 -warp_id 6 - xcfftool -dbname test_$cam -revert -xcff_id 1 -warp_id 7 - xcfftool -dbname test_$cam -revert -xcff_id 1 -warp_id 8 - xcfftool -dbname test_$cam -updatesummary -xcff_id 1 -fault 1 - xcfftool -dbname test_$cam -revertsummary -xccal_id 1 -endif + + # get a list of the unique xcff_ids + set xcff_ids_u = (`grep xcff_id xcff.todo.md | awk '{print $3}' | sort | uniq`) + + set n = 1 + while ($n <= $#xcff_ids_u) + # get the workdir from above which matches this uniq xcff_ids: + set workdir_u = "" + set skycell_u = "" + set xccal_u = "" + set t = 1 + while ($t <= $#xcff_ids) + if ($xcff_ids[$t] == $xcff_ids_u[$n]) then + set workdir_u = $workdir[$t] + set skycell_u = $Tskycells[$t] + set xccal_u = $xccal_ids[$t] + set xcff_u = $xcff_ids_u[$n] + endif + @ t++ + end + if ($workdir_u == "") then + echo "ERROR: failed to find workdir for $xcff_ids_u[$n] from $workdir" + exit 2 + endif + + # OUTROOT is defined in xcff.pro based on workdir : "%s/%s/%s/%s.%s.xcff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $XCFF_ID **** TBD *** + ## (pretend to) run xcff_summary.pl + xcfftool -dbname test_$cam -toadvance -xcff_id $xcff_u + + if (1 && ($xccal_u == 1)) then + mkdir -p $workdir[$n]/Fake.Tess/$Tskycells[$n] + echo xcff_summary.pl --dbname test_$cam --xcff_id $xcff_u --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].$cam.xcff.$xcff_u --camera GPC1 + xcff_summary.pl --dbname test_$cam --xcff_id $xcff_u --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].$cam.xcff.$xcff_u --camera GPC1 + else + xcfftool -dbname test_$cam -addsummary -xcff_id $xcff_u -path_base $workdir_u/Fake.Tess/$skycell_u/Fake.Tess.$skycell_u.$cam.xcff.$xcff_u -dtime_script 0.0 -quality 0 -fault 0 -hostname elepaio + endif + xcfftool -dbname test_$cam -summary -xcff_id $xcff_u + @ n++ + end + end + + # test the revert process + if (0) then + xcfftool -dbname test_$cam -updateresult -xcff_id 1 -warp_id 6 -fault 1 + xcfftool -dbname test_$cam -updateresult -xcff_id 1 -warp_id 7 -fault 1 + xcfftool -dbname test_$cam -updateresult -xcff_id 1 -warp_id 8 -fault 1 + xcfftool -dbname test_$cam -revert -xcff_id 1 -warp_id 6 + xcfftool -dbname test_$cam -revert -xcff_id 1 -warp_id 7 + xcfftool -dbname test_$cam -revert -xcff_id 1 -warp_id 8 + xcfftool -dbname test_$cam -updatesummary -xcff_id 1 -fault 1 + xcfftool -dbname test_$cam -revertsummary -xcff_id 1 + endif + +end # -exportrun list summary results # -importrun list summary results -exit 2 +exit 0 Index: /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/run.addstar.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/run.addstar.sh (revision 42611) +++ /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/run.addstar.sh (revision 42611) @@ -0,0 +1,7 @@ +#!/bin/csh -f + +# this works for the first warp from each camera, but fails for the +# rest because they are all duplicates in reality +foreach f ( xcff/Fake.Tess/skycell.2479.024/*.wrp.*.cmf ) + addstar -D CATDIR catdir.t2 -D CAMERA gpc1 -accept-astrom $f +end