Index: /branches/eam_branches/ipp-20140717/ippToPsps/test/fulltest.sh =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/test/fulltest.sh (revision 37162) +++ /branches/eam_branches/ipp-20140717/ippToPsps/test/fulltest.sh (revision 37163) @@ -1,5 +1,5 @@ #!/bin/csh -f -chmod +x *.dvo *.sh +chmod +x mkgpc1data.dvo mkgpc1.sh # a bit crude : get the datadir path by interpollation of the PATH element for psconfig @@ -14,4 +14,5 @@ # before running this test, you must have a mysql user: # mkgpc1.sh user localhost eam eam + set mkgpc1 = 1 set initdb = 1 @@ -157,5 +158,4 @@ mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_test" mysql -h localhost -u dvo -pdvo ipptopsps_test < $IPPTOPSPS_DATA/ippToPspsDbSchema.sql - endif Index: /branches/eam_branches/ipp-20140717/ippToPsps/test/mksasdata.dvo =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/test/mksasdata.dvo (revision 37162) +++ /branches/eam_branches/ipp-20140717/ippToPsps/test/mksasdata.dvo (revision 37163) @@ -4,10 +4,12 @@ if (not($?VERBOSE)) set VERBOSE = 0 +if (not($?PARALLEL)) set PARALLEL = 0 -$OUTDIR = sas12 +$OUTDIR = sasdata macro mkfull init.db - mkcatdir + # mkcatdir.cam + mkcatdir.stk end @@ -18,12 +20,7 @@ $dbpass = dvo - # exposure center - $RA = 333.8 - $DEC = 1.6 - - # stack center - # XXX the selected stack centers come from the input exposures, so I should use a gpc1 db with stacks associated with real exposures - $RA = 331.6 - $DEC = -6.0 + # are these used to find the exposures in any way? + $RA = 333.5 + $DEC = -5.0 break -auto off @@ -40,5 +37,5 @@ # create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp -macro mkcatdir +macro mkcatdir.cam if ($0 != 1) echo "mkcatdir" @@ -49,14 +46,14 @@ local catdir - $catdirbase = catdir.sas12 + $catdirbase = catdir.sas.cam $catdir = $OUTDIR/$catdirbase exec rm -rf $catdir - $expfile = sas12.test.psps/o5745g0620o.356996.cm.473458 + $expfile = sasdata/o5745g0620o.356996.cm.473458 exec addstar -D CAMERA gpc1 -D CATDIR $catdir $expfile.smf # are these used to find the exposures in any way? - $RA = 333.8 - $DEC = 1.6 + $RA = 333.5 + $DEC = -5.0 # a single exposure @@ -71,16 +68,4 @@ dbinsert addRun (stage, state, stage_id, minidvodb_name) values ("cam", "full", $expID, "$catdirbase") - # a single stack - $stkfile = sas12.test.psps/RINGS.V3.skycell.1225.070.stk.1032983.skycal.40531 - exec addstar -D CAMERA gpc1 -D CATDIR $catdir $stkfile.cmf - - $stkID = 1 - dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "skycell.0000.000", "g.00000", "testsoft") - dbinsert stackInputSkyfile (stack_id, warp_id) values ($stkID, $expID) - dbinsert stackSumSkyfile (stack_id) values ($stkID) - dbinsert skycalRun (skycal_id, stack_id) values ($stkID, $stkID) - dbinsert skycalResult (skycal_id, path_base) values ($stkID, "$stkfile") - dbinsert addRun (stage, state, stage_id, minidvodb_name) values ("skycal", "full", $stkID, "$catdirbase") - # catdirs for the exposures dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("$catdirbase", 1, "merged") @@ -89,4 +74,48 @@ dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (1, 1, "full", "$catdirbase") dbinsert mergedvodbProcessed (merge_id, fault) values (1, 0) +end + +# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp +macro mkcatdir.stk + if ($0 != 1) + echo "mkcatdir" + break + end + + mkdir $OUTDIR + + local catdir + $catdirbase = catdir.sas.stk + $catdir = $OUTDIR/$catdirbase + exec rm -rf $catdir + + # are these used to find the exposures in any way? + $RA = 333.5 + $DEC = -5.0 + + $TIMEFORMAT = mjd + $TIMEREF = 2001/01/01,00:00:00 + ctimes -abs 2010/01/01,00:00:00 -var mjd + + # a single stack + $stkfile = sasdata/RINGS.V3.skycell.1225.070.stk.3366216.skycal.1589960 + exec addstar -D CAMERA gpc1 -D CATDIR $catdir $stkfile.cmf + + # echo relphot -D CATDIR $catdir -region {$RA-5} {$RA+5} {$DEC-5} {$DEC+5} -images g,r,i,z,y -update + # exec relphot -D CATDIR $catdir -region {$RA-5} {$RA+5} {$DEC-5} {$DEC+5} -images g,r,i,z,y -update + + break -auto off + $tmpline = `echo $stkfile.cmf | fields IMAGEID` + list words -split $tmpline + $stkID = $words:1 + break -auto on + + $expID = 1 + dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "skycell.0000.000", "g.00000", "testsoft") + dbinsert stackInputSkyfile (stack_id, warp_id) values ($stkID, $expID) + dbinsert stackSumSkyfile (stack_id, mjd_obs) values ($stkID, $mjd) + dbinsert skycalRun (sky_id, skycal_id, stack_id) values ($stkID, $stkID, $stkID) + dbinsert skycalResult (skycal_id, path_base) values ($stkID, "$stkfile") + dbinsert addRun (stage, state, stage_id, minidvodb_name) values ("skycal", "full", $stkID, "$catdirbase") # catdirs for the stack Index: /branches/eam_branches/ipp-20140717/ippToPsps/test/sastest.sh =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/test/sastest.sh (revision 37162) +++ /branches/eam_branches/ipp-20140717/ippToPsps/test/sastest.sh (revision 37163) @@ -1,9 +1,5 @@ #!/bin/csh -f -set stackqueuename = 0 -if ($#argv > 0) then - set stackqueuename = $1 - echo "stackqueue: $stackqueuename" -endif +chmod +x mksasdata.dvo mkgpc1.sh # a bit crude : get the datadir path by interpolation of the PATH element for psconfig @@ -19,18 +15,136 @@ # mkgpc1.sh user localhost eam eam -set mkgpc1 = 1 -set initdb = 1 -set mkdummy = 1 -set initscratch = 1 -set initbatch = 1 -set camqueue = 1 -set cambatch = 1 -set stackqueue = 0 -set stackbatch = 0 - -set OUTDIR = "sas12" +set mkgpc1 = 1 +set initdb = 1 +set mkdummy = 1 +set initscratch = 1 +set initbatch = 1 +set camqueue = 1 +set cambatch = 1 +set stackqueue = 1 +set stackbatch = 1 +set objectqueue = 1 +set objectbatch = 1 + +set targ = ($argv) +set args = "" + +while ($#targ) + switch ("$targ[1]") + case -h + case -help + echo "USAGE: fulltest.sh [-XX] [+XX]" + echo " -XX : turn off features" + echo " +XX : turn on features" + echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, object" + exit 2 + case -all + set mkgpc1 = 0 + set initdb = 0 + set initscratch = 0 + set initbatch = 0 + set camqueue = 0 + set cambatch = 0 + set stackqueue = 0 + set stackbatch = 0 + set objectqueue = 0 + set objectbatch = 0 + breaksw; + case -mkgpc1 + set mkgpc1 = 0 + breaksw; + case -initdb + set initdb = 0 + breaksw; + case -initscratch + set initscratch = 0 + breaksw; + case -initbatch + set initbatch = 0 + breaksw; + case -camera + set camqueue = 0 + set cambatch = 0 + breaksw; + case -stack + set stackqueue = 0 + set stackbatch = 0 + breaksw; + case -object + set objectqueue = 0 + set objectbatch = 0 + breaksw; + default: + set args=($args $targ[1]); + breaksw; + endsw + shift targ +end + +set targ = ($args) +set args = "" + +while ($#targ) + switch ("$targ[1]") + case +all + set mkgpc1 = 1 + set initdb = 1 + set initscratch = 1 + set initbatch = 1 + set camqueue = 1 + set cambatch = 1 + set stackqueue = 1 + set stackbatch = 1 + set objectqueue = 1 + set objectbatch = 1 + breaksw; + case +mkgpc1 + set mkgpc1 = 1 + breaksw; + case +initdb + set initdb = 1 + breaksw; + case +initscratch + set initscratch = 1 + breaksw; + case +initbatch + set initbatch = 1 + breaksw; + case "+camera" + set camqueue = 1 + set cambatch = 1 + breaksw; + case +stack + set stackqueue = 1 + set stackbatch = 1 + breaksw; + case +object + set objectqueue = 1 + set objectbatch = 1 + breaksw; + default: + set args=($args $targ[1]); + breaksw; + endsw + shift targ +end + +if ($args[1] != "") then + echo "USAGE: sastest.sh [-XX] [+XX]" + echo " -XX : turn off features" + echo " +XX : turn on features" + echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, object" + exit 2 +endif + +set stackqueuename = 0 +set objectqueuename = 0 + +set OUTDIR = "sasdata" set queuename = "test" if ($mkgpc1) then + echo "" + echo " ---- mksasdata.dvo : creating the dvo & gpc1 database entries ----" mksasdata.dvo endif @@ -38,4 +152,6 @@ # create the ipptopsps database if ($initdb) then + echo "" + echo " ---- init ipptopsps_test ----" # does the database already exist? @@ -43,5 +159,4 @@ mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_test" mysql -h localhost -u dvo -pdvo ipptopsps_test < $IPPTOPSPS_DATA/ippToPspsDbSchema.sql - endif @@ -53,88 +168,151 @@ mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_scratch" ippjython setupScratchDb.py -test + if ($status) exit +endif + +# this is poor : we need to queue something in order to generate an init batch +# this is because a skychunk carries the info about the datastore used to publish +# the batch +if ($initbatch) then + echo "" + echo " ---- make skychunk for IN batches ----" + + echo "initbatch " > initbatch.dat # skychunk name + echo "test_ds " >> initbatch.dat # datastore_product + echo "IPP_PSPS " >> initbatch.dat # datastore_type + echo "0 " >> initbatch.dat # datastore_publish + echo "none " >> initbatch.dat # dvo_label + echo "none " >> initbatch.dat # dvo_location + echo "0 " >> initbatch.dat # min_ra + echo "0 " >> initbatch.dat # max_ra + echo "0 " >> initbatch.dat # min_dec + echo "0 " >> initbatch.dat # max_dec + echo "2 " >> initbatch.dat # box_size + echo "none " >> initbatch.dat # base_path + echo "0 " >> initbatch.dat # data_release + echo "0 " >> initbatch.dat # delete_local + echo "1 " >> initbatch.dat # delete_datastore + echo "1 " >> initbatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> initbatch.dat # epoch + echo "3PI " >> initbatch.dat # survey + echo "3PI " >> initbatch.dat # psps_survey + echo "0 " >> initbatch.dat # queue_P2 + echo "0 " >> initbatch.dat # queue_ST + echo "0 " >> initbatch.dat # queue_OB + echo "0 " >> initbatch.dat # queue_DF + echo "0 " >> initbatch.dat # queue_DO + echo "0 " >> initbatch.dat # queue_FW + echo "0 " >> initbatch.dat # queue_FO + echo "1 " >> initbatch.dat # active + echo "0 " >> initbatch.dat # parallel + echo "use_new " >> initbatch.dat # P2_smf_version + echo "any " >> initbatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> initbatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> initbatch.dat # trange_end + cat initbatch.dat | ippjython queue.py -test edit + if ($status) exit + + echo "" + echo " ---- create init batch ----" + + rm -rf $OUTDIR/IN + ippjython loader.py -test initbatch init + if ($status) exit endif # this code is sensitive to a previously existing database? if ($camqueue) then - ippjython queue.py -test edit < cambatch.dat # skychunk name + echo "test_ds " >> cambatch.dat # datastore_product + echo "IPP_PSPS " >> cambatch.dat # datastore_type + echo "0 " >> cambatch.dat # datastore_publish + echo "catdir.sas.cam " >> cambatch.dat # dvo_label + echo "$OUTDIR/catdir.sas.cam" >> cambatch.dat # dvo_location + echo "330 " >> cambatch.dat # min_ra + echo "340 " >> cambatch.dat # max_ra + echo "-8 " >> cambatch.dat # min_dec + echo "2 " >> cambatch.dat # max_dec + echo "1 " >> cambatch.dat # box_size + echo "$OUTDIR " >> cambatch.dat # base_path + echo "0 " >> cambatch.dat # data_release + echo "0 " >> cambatch.dat # delete_local + echo "1 " >> cambatch.dat # delete_datastore + echo "1 " >> cambatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> cambatch.dat # epoch + echo "3PI " >> cambatch.dat # survey + echo "3PI " >> cambatch.dat # psps_survey + echo "1 " >> cambatch.dat # queue_P2 + echo "0 " >> cambatch.dat # queue_ST + echo "0 " >> cambatch.dat # queue_OB + echo "0 " >> cambatch.dat # queue_DF + echo "0 " >> cambatch.dat # queue_DO + echo "0 " >> cambatch.dat # queue_FW + echo "0 " >> cambatch.dat # queue_FO + echo "1 " >> cambatch.dat # active + echo "0 " >> cambatch.dat # parallel + echo "use_new " >> cambatch.dat # P2_smf_version + echo "any " >> cambatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> cambatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> cambatch.dat # trange_end + cat cambatch.dat | ippjython queue.py -test edit + if ($status) exit endif # this code is sensitive to a previously existing database? if ($stackqueue) then + echo "" + echo " ---- queue ST batches ----" + if ($stackqueuename == 0) then set stackqueuename = $queuename\_stk endif - ippjython queue.py -test edit < stackbatch.dat # skychunk name + echo "test_ds " >> stackbatch.dat # datastore_product + echo "IPP_PSPS " >> stackbatch.dat # datastore_type + echo "0 " >> stackbatch.dat # datastore_publish + echo "catdir.sas.stk " >> stackbatch.dat # dvo_label + echo "$OUTDIR/catdir.sas.stk" >> stackbatch.dat # dvo_location + echo "332 " >> stackbatch.dat # min_ra + echo "335 " >> stackbatch.dat # max_ra + echo "-6 " >> stackbatch.dat # min_dec + echo "-4 " >> stackbatch.dat # max_dec + echo "1 " >> stackbatch.dat # box_size + echo "$OUTDIR " >> stackbatch.dat # base_path + echo "0 " >> stackbatch.dat # data_release + echo "0 " >> stackbatch.dat # delete_local + echo "1 " >> stackbatch.dat # delete_datastore + echo "1 " >> stackbatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> stackbatch.dat # epoch + echo "3PI " >> stackbatch.dat # survey + echo "3PI " >> stackbatch.dat # psps_survey + echo "0 " >> stackbatch.dat # queue_P2 + echo "1 " >> stackbatch.dat # queue_ST + echo "0 " >> stackbatch.dat # queue_OB + echo "0 " >> stackbatch.dat # queue_DF + echo "0 " >> stackbatch.dat # queue_DO + echo "0 " >> stackbatch.dat # queue_FW + echo "0 " >> stackbatch.dat # queue_FO + echo "1 " >> stackbatch.dat # active + echo "0 " >> stackbatch.dat # parallel + echo "use_new " >> stackbatch.dat # P2_smf_version + echo "any " >> stackbatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> stackbatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> stackbatch.dat # trange_end + cat stackbatch.dat | ippjython queue.py -test edit + if ($status) exit endif if ($stackbatch) then + echo "" + echo " ---- load ST batches ----" + if ($stackqueuename == 0) then set stackqueuename = $queuename\_stk endif + rm -rf $OUTDIR/ST ippjython loader.py -test $stackqueuename once -endif - + if ($status) exit +endif