Index: trunk/ippToPsps/test/fulltest.sh =================================================================== --- trunk/ippToPsps/test/fulltest.sh (revision 35753) +++ trunk/ippToPsps/test/fulltest.sh (revision 37246) @@ -1,5 +1,5 @@ #!/bin/csh -f -chmod +x *.dvo *.sh +chmod +x mkgpc1data.dvo mkgpc1.sh # a bit crude : get the datadir path by interpollation of the PATH element for psconfig @@ -14,4 +14,5 @@ # before running this test, you must have a mysql user: # mkgpc1.sh user localhost eam eam + set mkgpc1 = 1 set initdb = 1 @@ -157,5 +158,4 @@ mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_test" mysql -h localhost -u dvo -pdvo ipptopsps_test < $IPPTOPSPS_DATA/ippToPspsDbSchema.sql - endif @@ -168,31 +168,7 @@ mysql -h localhost -u dvo -pdvo -e "drop database ipptopsps_test_scratch" mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_test_scratch" - ippjython setupScratchDb.py -test -endif - -# `name` -# `datastore_product` -# `datastore_type` -# `datastore_publish` -# `dvo_label` -# `dvo_location` -# `min_ra` -# `max_ra` -# `min_dec` -# `max_dec` -# `box_size` -# `base_path` -# `data_release` -# `delete_local` -# `delete_datastore` -# `delete_dxlayer` -# `epoch` -# `survey` -# `psps_survey` -# `queue_P2` -# `queue_ST` -# `queue_OB` -# `active` -# `parallel` + ippjython setupScratchDb.py -test < /dev/null + if ($status) exit +endif # this is poor : we need to queue something in order to generate an init batch @@ -203,32 +179,38 @@ echo " ---- make skychunk for IN batches ----" - ippjython queue.py -test edit < initbatch.dat # skychunk name + echo "test_ds " >> initbatch.dat # datastore_product + echo "IPP_PSPS " >> initbatch.dat # datastore_type + echo "0 " >> initbatch.dat # datastore_publish + echo "none " >> initbatch.dat # dvo_label + echo "none " >> initbatch.dat # dvo_location + echo "0 " >> initbatch.dat # min_ra + echo "0 " >> initbatch.dat # max_ra + echo "0 " >> initbatch.dat # min_dec + echo "0 " >> initbatch.dat # max_dec + echo "2 " >> initbatch.dat # box_size + echo "none " >> initbatch.dat # base_path + echo "0 " >> initbatch.dat # data_release + echo "0 " >> initbatch.dat # delete_local + echo "1 " >> initbatch.dat # delete_datastore + echo "1 " >> initbatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> initbatch.dat # epoch + echo "3PI " >> initbatch.dat # survey + echo "3PI " >> initbatch.dat # psps_survey + echo "0 " >> initbatch.dat # queue_P2 + echo "0 " >> initbatch.dat # queue_ST + echo "0 " >> initbatch.dat # queue_OB + echo "0 " >> initbatch.dat # queue_DF + echo "0 " >> initbatch.dat # queue_DO + echo "0 " >> initbatch.dat # queue_FW + echo "0 " >> initbatch.dat # queue_FO + echo "1 " >> initbatch.dat # active + echo "0 " >> initbatch.dat # parallel + echo "use_new " >> initbatch.dat # P2_smf_version + echo "any " >> initbatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> initbatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> initbatch.dat # trange_end + cat initbatch.dat | ippjython queue.py -test edit + if ($status) exit echo "" @@ -236,5 +218,6 @@ rm -rf $OUTDIR/IN - ippjython loader.py -test initbatch init + ippjython loader.py -test initbatch init -simtest < /dev/null + if ($status) exit endif @@ -244,32 +227,38 @@ echo " ---- queue P2 batches ----" - ippjython queue.py -test edit < cambatch.dat # skychunk name + echo "test_ds " >> cambatch.dat # datastore_product + echo "IPP_PSPS " >> cambatch.dat # datastore_type + echo "0 " >> cambatch.dat # datastore_publish + echo "catdir.cam " >> cambatch.dat # dvo_label + echo "$OUTDIR/catdir.cam " >> cambatch.dat # dvo_location + echo "9 " >> cambatch.dat # min_ra + echo "11 " >> cambatch.dat # max_ra + echo "19 " >> cambatch.dat # min_dec + echo "21 " >> cambatch.dat # max_dec + echo "2 " >> cambatch.dat # box_size + echo "$OUTDIR " >> cambatch.dat # base_path + echo "0 " >> cambatch.dat # data_release + echo "0 " >> cambatch.dat # delete_local + echo "1 " >> cambatch.dat # delete_datastore + echo "1 " >> cambatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> cambatch.dat # epoch + echo "3PI " >> cambatch.dat # survey + echo "3PI " >> cambatch.dat # psps_survey + echo "1 " >> cambatch.dat # queue_P2 + echo "0 " >> cambatch.dat # queue_ST + echo "0 " >> cambatch.dat # queue_OB + echo "0 " >> cambatch.dat # queue_DF + echo "0 " >> cambatch.dat # queue_DO + echo "0 " >> cambatch.dat # queue_FW + echo "0 " >> cambatch.dat # queue_FO + echo "1 " >> cambatch.dat # active + echo "0 " >> cambatch.dat # parallel + echo "use_new " >> cambatch.dat # P2_smf_version + echo "any " >> cambatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> cambatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> cambatch.dat # trange_end + cat cambatch.dat | ippjython queue.py -test edit -simtest + if ($status) exit endif @@ -279,5 +268,6 @@ rm -rf $OUTDIR/P2 - ippjython loader.py -test $queuename once + ippjython loader.py -test $queuename once -simtest < /dev/null + if ($status) exit endif @@ -290,32 +280,38 @@ set stackqueuename = $queuename\_stk endif - ippjython queue.py -test edit < stackbatch.dat # skychunk name + echo "test_ds " >> stackbatch.dat # datastore_product + echo "IPP_PSPS " >> stackbatch.dat # datastore_type + echo "0 " >> stackbatch.dat # datastore_publish + echo "catdir.stk " >> stackbatch.dat # dvo_label + echo "$OUTDIR/catdir.stk " >> stackbatch.dat # dvo_location + echo "9 " >> stackbatch.dat # min_ra + echo "11 " >> stackbatch.dat # max_ra + echo "19 " >> stackbatch.dat # min_dec + echo "21 " >> stackbatch.dat # max_dec + echo "2 " >> stackbatch.dat # box_size + echo "$OUTDIR " >> stackbatch.dat # base_path + echo "0 " >> stackbatch.dat # data_release + echo "0 " >> stackbatch.dat # delete_local + echo "1 " >> stackbatch.dat # delete_datastore + echo "1 " >> stackbatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> stackbatch.dat # epoch + echo "3PI " >> stackbatch.dat # survey + echo "3PI " >> stackbatch.dat # psps_survey + echo "0 " >> stackbatch.dat # queue_P2 + echo "1 " >> stackbatch.dat # queue_ST + echo "0 " >> stackbatch.dat # queue_OB + echo "0 " >> stackbatch.dat # queue_DF + echo "0 " >> stackbatch.dat # queue_DO + echo "0 " >> stackbatch.dat # queue_FW + echo "0 " >> stackbatch.dat # queue_FO + echo "1 " >> stackbatch.dat # active + echo "0 " >> stackbatch.dat # parallel + echo "use_new " >> stackbatch.dat # P2_smf_version + echo "any " >> stackbatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> stackbatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> stackbatch.dat # trange_end + cat stackbatch.dat | ippjython queue.py -test edit -simtest + if ($status) exit endif @@ -328,5 +324,6 @@ endif rm -rf $OUTDIR/ST - ippjython loader.py -test $stackqueuename once + ippjython loader.py -test $stackqueuename once -simtest < /dev/null + if ($status) exit endif @@ -339,32 +336,38 @@ set objectqueuename = $queuename\_obj endif - ippjython queue.py -test edit < objectbatch.dat # skychunk name + echo "test_ds " >> objectbatch.dat # datastore_product + echo "IPP_PSPS " >> objectbatch.dat # datastore_type + echo "0 " >> objectbatch.dat # datastore_publish + echo "catdir.cam " >> objectbatch.dat # dvo_label + echo "$OUTDIR/catdir.cam " >> objectbatch.dat # dvo_location + echo "9 " >> objectbatch.dat # min_ra + echo "11 " >> objectbatch.dat # max_ra + echo "19 " >> objectbatch.dat # min_dec + echo "21 " >> objectbatch.dat # max_dec + echo "2 " >> objectbatch.dat # box_size + echo "$OUTDIR " >> objectbatch.dat # base_path + echo "0 " >> objectbatch.dat # data_release + echo "0 " >> objectbatch.dat # delete_local + echo "1 " >> objectbatch.dat # delete_datastore + echo "1 " >> objectbatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> objectbatch.dat # epoch + echo "3PI " >> objectbatch.dat # survey + echo "3PI " >> objectbatch.dat # psps_survey + echo "0 " >> objectbatch.dat # queue_P2 + echo "0 " >> objectbatch.dat # queue_ST + echo "1 " >> objectbatch.dat # queue_OB + echo "0 " >> objectbatch.dat # queue_DF + echo "0 " >> objectbatch.dat # queue_DO + echo "0 " >> objectbatch.dat # queue_FW + echo "0 " >> objectbatch.dat # queue_FO + echo "1 " >> objectbatch.dat # active + echo "0 " >> objectbatch.dat # parallel + echo "use_new " >> objectbatch.dat # P2_smf_version + echo "any " >> objectbatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> objectbatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> objectbatch.dat # trange_end + cat objectbatch.dat | ippjython queue.py -test edit once -simtest + if ($status) exit endif @@ -377,5 +380,7 @@ endif rm -rf $OUTDIR/OB - ippjython loader.py -test $objectqueuename once -endif - + + ippjython loader.py -test $objectqueuename once -simtest < /dev/null + if ($status) exit +endif + Index: trunk/ippToPsps/test/mkgpc1.sh =================================================================== --- trunk/ippToPsps/test/mkgpc1.sh (revision 35753) +++ trunk/ippToPsps/test/mkgpc1.sh (revision 37246) @@ -47,5 +47,6 @@ bg_stdev REAL, ra FLOAT, - decl FLOAT + decl FLOAT, + dateobs DATETIME ); CREATE TABLE chipRun ( @@ -88,7 +89,9 @@ ); CREATE TABLE stackSumSkyfile ( - stack_id INT + stack_id INT, + mjd_obs DOUBLE ); CREATE TABLE skycalRun ( + sky_id INT, skycal_id INT, stack_id INT @@ -155,5 +158,5 @@ echo ntable: $ntable if ($ntable != 21) then - echo "expected 21 lines (19 tables), but got $ntable, not deleting" + echo "expected 21 lines (20 tables), but got $ntable, not deleting" exit 2 endif Index: trunk/ippToPsps/test/mkgpc1data.dvo =================================================================== --- trunk/ippToPsps/test/mkgpc1data.dvo (revision 35753) +++ trunk/ippToPsps/test/mkgpc1data.dvo (revision 37246) @@ -15,16 +15,16 @@ # exp_id stack_id raw_file exp_file offset list imagedata - 1 1 $OUTDIR/test.01 0.000 - 2 1 $OUTDIR/test.02 -0.025 - 3 1 $OUTDIR/test.03 0.025 - 4 1 $OUTDIR/test.04 0.010 - 5 2 $OUTDIR/test.05 0.000 - 6 2 $OUTDIR/test.06 -0.025 - 7 2 $OUTDIR/test.07 0.025 - 8 2 $OUTDIR/test.08 0.010 - 9 3 $OUTDIR/test.09 0.000 -10 3 $OUTDIR/test.10 -0.025 -11 3 $OUTDIR/test.11 0.025 -12 3 $OUTDIR/test.12 0.010 + 1 1 $OUTDIR/test.01 0.000 2010/01/01 01:00:00 + 2 1 $OUTDIR/test.02 -0.025 2010/01/01 02:00:00 + 3 1 $OUTDIR/test.03 0.025 2010/01/01 03:00:00 + 4 1 $OUTDIR/test.04 0.010 2010/01/01 04:00:00 + 5 2 $OUTDIR/test.05 0.000 2010/01/01 05:00:00 + 6 2 $OUTDIR/test.06 -0.025 2010/01/01 06:00:00 + 7 2 $OUTDIR/test.07 0.025 2010/01/01 07:00:00 + 8 2 $OUTDIR/test.08 0.010 2010/01/01 08:00:00 + 9 3 $OUTDIR/test.09 0.000 2010/01/01 09:00:00 +10 3 $OUTDIR/test.10 -0.025 2010/01/01 10:00:00 +11 3 $OUTDIR/test.11 0.025 2010/01/01 11:00:00 +12 3 $OUTDIR/test.12 0.010 2010/01/01 12:00:00 end @@ -39,6 +39,8 @@ # mkcatdir.cam PS1_V3 PS1_V4 # mkcatdir.stk PS1_V3 PS1_V4 - mkcatdir.cam PS1_V4 PS1_V4 - mkcatdir.stk PS1_V4 PS1_V4 + # mkcatdir.cam PS1_V4 PS1_V4 + # mkcatdir.stk PS1_V4 PS1_V4 + mkcatdir.cam PS1_V5 PS1_V5 + mkcatdir.stk PS1_V5 PS1_V5 insert.stack.set end @@ -94,5 +96,8 @@ $offset = $word:3 - ctimes -abs 2008/01/01,$i\:00:00 -var mjd + sprintf myDATE "%s" $word:4 + sprintf myTIME "%s" $word:5 + echo ctimes -abs $myDATE,$myTIME -var mjd + ctimes -abs $myDATE,$myTIME -var mjd # XXX for a simple test of ippToPsps, i need to generate smf files with some correspondence to gpc1 exposure smfs @@ -142,5 +147,5 @@ mkinput $offset $rawfile - exec mkcmf -no-noise -append $rawfile $cmffile -date 2008/1/1 -time $i\:00:00 -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 + exec mkcmf -photcode SIMTEST.r.Chip -no-noise -append $rawfile $cmffile -date $myDATE -time $myTIME -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 # keywords which are read from image extension @@ -203,20 +208,23 @@ end - if ($i == 5) + if (0 && ($i == 5)) echo "NOTE: adding image $i 2x to DVO, but without detections : this simulates a failure in LAP dvo" - echo addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -image -dup-images - exec addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -image -dup-images + echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -image -dup-images + exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -image -dup-images end - if ($i == 7) + if (0 && ($i == 7)) echo "NOTE: adding detections from image $i 2x to DVO, but with wrong photcode and duplicate image ID : this simulates a failure in LAP dvo" - echo addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -dup-images -image-id-override 7 -photcode 2MASS_J - exec addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -dup-images -image-id-override 7 -photcode 2MASS_J + echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -dup-images -image-id-override 7 -photcode 2MASS_J + exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -dup-images -image-id-override 7 -photcode 2MASS_J end - echo addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass - exec addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass - end + echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass + exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass + end + + echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update + exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update if ($PARALLEL) @@ -264,13 +272,21 @@ $offset = $word:2 + ## this is not generating photcodes recognized as stack by relphot + mkinput $offset $rawfile - exec mkcmf $rawfile $cmffile -extname SkyChip -date 2008/1/1 -time $i\:00:00 -radec $RA $DEC -type $1 -imageID $ID -sourceID 1 + exec mkcmf -photcode SIMTEST.r.SkyChip $rawfile $cmffile -extname SkyChip -date 2008/1/1 -time $i\:00:00 -radec $RA $DEC -type $1 -imageID $ID -sourceID 1 # add some required header keywords: exec echo "HIERARCH FPA.ZP = 25. / Magnitude zero point" > tmp.hdr + exec echo "TESS_ID = 'RINGS.V3'" >> tmp.hdr + exec echo "SKYCELL = 'skycell.1133.081'" >> tmp.hdr exec fits_insert $cmffile tmp.hdr - exec addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass - end + echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass + exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass + end + + echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update + exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update if ($PARALLEL) @@ -308,7 +324,10 @@ # a single exposure - dbinsert rawExp (exp_id, exp_name, exp_time, bg, bg_stdev, ra, decl) values ($myID, "test.$myID", 10.0, 100.0, 2.0, {$RA*3.1415/180.0}, {$DEC*3.1415/180.0}) + echo $myDATE + echo $myTIME + + dbinsert rawExp (exp_id, exp_name, exp_time, bg, bg_stdev, ra, decl, dateobs) values ($myID, "test.$myID", 10.0, 100.0, 2.0, {$RA*3.1415/180.0}, {$DEC*3.1415/180.0}, '$myDATE $myTIME') dbinsert chipRun (exp_id, chip_id) values ($myID, $myID) - dbinsert camRun (chip_id, cam_id, dist_group, software_ver) values ($myID, $myID, "testgroup", "testsoft") + dbinsert camRun (chip_id, cam_id, dist_group, software_ver) values ($myID, $myID, "testgroup", "38000M") dbinsert camProcessedExp (cam_id, path_base, fault) values ($myID, "$filename", 0) dbinsert fakeRun (fake_id, cam_id) values ($myID, $myID) @@ -332,5 +351,11 @@ $stkfile = $2 - dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "skycell.0000.000", "g.00000", "testsoft") + dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "skycell.0000.000", "g.00000", "37500") + + $TIMEFORMAT = mjd + $TIMEREF = 2001/01/01,00:00:00 + + $mjdsum = 0.0 + $Nstk = 0 for i 0 $imagedata:n @@ -341,9 +366,18 @@ $expID = $tmp:0 $expfile = $tmp:2 + sprintf myDATE "%s" $tmp:4 + sprintf myTIME "%s" $tmp:5 insert.exp $expID $expfile dbinsert stackInputSkyfile (stack_id, warp_id) values ($stkID, $expID) - end - dbinsert stackSumSkyfile (stack_id) values ($stkID) - dbinsert skycalRun (skycal_id, stack_id) values ($stkID, $stkID) + + echo ctimes -abs $myDATE,$myTIME -var mjd + ctimes -abs $myDATE,$myTIME -var mjd + $mjdsum += $mjd + $Nstk ++ + end + $mjd = $mjdsum / $Nstk + + dbinsert stackSumSkyfile (stack_id, mjd_obs) values ($stkID, $mjd) + dbinsert skycalRun (sky_id, skycal_id, stack_id) values ($stkID, $stkID, $stkID) dbinsert skycalResult (skycal_id, path_base) values ($stkID, "$stkfile") Index: trunk/ippToPsps/test/mksasdata.dvo =================================================================== --- trunk/ippToPsps/test/mksasdata.dvo (revision 35753) +++ trunk/ippToPsps/test/mksasdata.dvo (revision 37246) @@ -4,10 +4,24 @@ if (not($?VERBOSE)) set VERBOSE = 0 +if (not($?PARALLEL)) set PARALLEL = 0 -$OUTDIR = sas12 +$OUTDIR = sasdata + +if (not($?KEEPDVO)) set KEEPDVO = 0 + +list expfiles + sasdata/o5745g0495o.356867.cm.982612 + sasdata/o5745g0507o.356882.cm.982626 + sasdata/o5745g0517o.356892.cm.982636 + sasdata/o5745g0527o.356909.cm.982650 +end macro mkfull init.db - mkcatdir + mkcatdir.cam + mkcatdir.stk + mkcal + mkgpc1.cam + mkgpc1.stk end @@ -18,12 +32,8 @@ $dbpass = dvo - # exposure center - $RA = 333.8 - $DEC = 1.6 - - # stack center - # XXX the selected stack centers come from the input exposures, so I should use a gpc1 db with stacks associated with real exposures - $RA = 331.6 - $DEC = -6.0 + # are these used to find the exposures in any way? + # center of the test skycell + $RA = 333.5 + $DEC = -5.0 break -auto off @@ -36,9 +46,9 @@ # system-wide info dbinsert ippToPspsFake (my_id) values (2) - dbinsert skycell (radeg, decdeg, skycell_id) values ($RA, $DEC, "skycell.0000.000") + dbinsert skycell (radeg, decdeg, skycell_id) values ($RA, $DEC, "skycell.1225.070") end # create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp -macro mkcatdir +macro mkcatdir.cam if ($0 != 1) echo "mkcatdir" @@ -49,44 +59,122 @@ local catdir - $catdirbase = catdir.sas12 + $catdirbase = catdir.sas.cam + $catdir = $OUTDIR/$catdirbase + + exec rm -rf $catdir + + for i 0 $expfiles:n + $expfile = $expfiles:$i + + exec addstar -D CAMERA gpc1 -D CATDIR $catdir -D CATFORMAT PS1_V5 $expfile.smf -update + end + exec addstar -D CATDIR $catdir -region 330 340 -10 0 -resort +end + +# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp +macro mkcatdir.stk + if ($0 != 1) + echo "mkcatdir" + break + end + + mkdir $OUTDIR + + local catdir + $catdirbase = catdir.sas.stk $catdir = $OUTDIR/$catdirbase exec rm -rf $catdir - $expfile = sas12.test.psps/o5745g0620o.356996.cm.473458 - exec addstar -D CAMERA gpc1 -D CATDIR $catdir $expfile.smf + # a single stack + $stkfile = sasdata/RINGS.V3.skycell.1225.070.stk.3366216.skycal.1589960 + exec addstar -D CAMERA gpc1 -D CATFORMAT PS1_V5 -D CATDIR $catdir $stkfile.cmf +end - # are these used to find the exposures in any way? - $RA = 333.8 - $DEC = 1.6 +macro mkcal - # a single exposure - $expID = 1 - dbinsert rawExp (exp_id, exp_name, exp_time, bg, bg_stdev, ra, decl) values ($expID, "o5745g0620o", 10.0, 100.0, 2.0, {$RA*3.1415/180.0}, {$DEC*3.1415/180.0}) - dbinsert chipRun (exp_id, chip_id) values ($expID, $expID) - dbinsert camRun (chip_id, cam_id, dist_group, software_ver) values ($expID, $expID, "testgroup", "testsoft") - dbinsert camProcessedExp (cam_id, path_base) values ($expID, "$expfile") - dbinsert fakeRun (fake_id, cam_id) values ($expID, $expID) - dbinsert warpRun (warp_id, fake_id) values ($expID, $expID) - dbinsert warpSkyCellMap (warp_id, skycell_id, class_id) values ($expID, "skycell.0000.000", "XY00") - dbinsert addRun (stage, state, stage_id, minidvodb_name) values ("cam", "full", $expID, "$catdirbase") + local catdir + $catdir = $OUTDIR/catdir.sas.cam + + # merge the cam and stk databases together: + exec dvomerge $OUTDIR/catdir.sas.stk into $catdir + + # run relphot + echo relphot -D CATDIR $catdir -region 330 340 -10 0 -images g,r,i,z,y -update -boundary-tree tess.3pi.fits + exec relphot -D CATDIR $catdir -region 330 340 -10 0 -images g,r,i,z,y -update -boundary-tree tess.3pi.fits + + echo relastro -v -D CATDIR $catdir -images -update-chips -region 330 340 -10 0 -update + exec relastro -v -D CATDIR $catdir -images -update-chips -region 330 340 -10 0 -update +end + +# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp +macro mkgpc1.cam + if ($0 != 1) + echo "mkgpc1.cam" + break + end + + local catdirbase + $catdirbase = catdir.sas.cam + + for i 0 $expfiles:n + $expfile = $expfiles:$i + + # are these used to find the exposures in any way? + $RA = 333.9 + $DEC = -3.8 + + # a single exposure + $expID = $i + 1 + substr $expfile 8 11 expname + + dbinsert rawExp (exp_id, exp_name, exp_time, bg, bg_stdev, ra, decl, dateobs) values ($expID, "$expname", 10.0, 100.0, 2.0, {$RA*3.1415/180.0}, {$DEC*3.1415/180.0}, "2010/01/01 00:00:00") + dbinsert chipRun (exp_id, chip_id) values ($expID, $expID) + dbinsert camRun (chip_id, cam_id, dist_group, software_ver) values ($expID, $expID, "testgroup", "38000M") + dbinsert camProcessedExp (cam_id, path_base, fault) values ($expID, "$expfile", 0) + dbinsert fakeRun (fake_id, cam_id) values ($expID, $expID) + dbinsert warpRun (warp_id, fake_id) values ($expID, $expID) + dbinsert warpSkyCellMap (warp_id, skycell_id, class_id) values ($expID, "skycell.1225.070", "XY00") + dbinsert addRun (add_id, stage, state, stage_id, minidvodb_name) values ($expID, "cam", "full", $expID, "$catdirbase") + dbinsert addProcessedExp (add_id, fault) values ($expID, 0) + + # catdirs for the exposures + dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("$catdirbase", 1, "merged") + dbinsert minidvodbProcessed (minidvodb_id, fault) values (1, 0) + + dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (1, 1, "full", "$catdirbase") + dbinsert mergedvodbProcessed (merge_id, fault) values (1, 0) + end +end + +# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp +macro mkgpc1.stk + if ($0 != 1) + echo "mkgpc1.stk" + break + end + + local catdirbase + $catdirbase = catdir.sas.cam + + $TIMEFORMAT = mjd + $TIMEREF = 2001/01/01,00:00:00 + ctimes -abs 2010/01/01,00:00:00 -var mjd # a single stack - $stkfile = sas12.test.psps/RINGS.V3.skycell.1225.070.stk.1032983.skycal.40531 - exec addstar -D CAMERA gpc1 -D CATDIR $catdir $stkfile.cmf + $stkfile = sasdata/RINGS.V3.skycell.1225.070.stk.3366216.skycal.1589960 - $stkID = 1 - dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "skycell.0000.000", "g.00000", "testsoft") + break -auto off + $tmpline = `echo $stkfile.cmf | fields IMAGEID` + list words -split $tmpline + $stkID = $words:1 + break -auto on + + $expID = 1 + dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "skycell.1225.070", "i.00000", "37500M") dbinsert stackInputSkyfile (stack_id, warp_id) values ($stkID, $expID) - dbinsert stackSumSkyfile (stack_id) values ($stkID) - dbinsert skycalRun (skycal_id, stack_id) values ($stkID, $stkID) + dbinsert stackSumSkyfile (stack_id, mjd_obs) values ($stkID, $mjd) + dbinsert skycalRun (sky_id, skycal_id, stack_id) values ($stkID, $stkID, $stkID) dbinsert skycalResult (skycal_id, path_base) values ($stkID, "$stkfile") dbinsert addRun (stage, state, stage_id, minidvodb_name) values ("skycal", "full", $stkID, "$catdirbase") - - # catdirs for the exposures - dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("$catdirbase", 1, "merged") - dbinsert minidvodbProcessed (minidvodb_id, fault) values (1, 0) - - dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (1, 1, "full", "$catdirbase") - dbinsert mergedvodbProcessed (merge_id, fault) values (1, 0) # catdirs for the stack Index: trunk/ippToPsps/test/sascheck.v0.sql =================================================================== --- trunk/ippToPsps/test/sascheck.v0.sql (revision 37246) +++ trunk/ippToPsps/test/sascheck.v0.sql (revision 37246) @@ -0,0 +1,25 @@ + +-- get a set of measurements matched by detID +select + gra, + gdec, + objID, + HEX(ippObjID), + surveyID, + tessID, + projectionID, + skyCellID, + randomStackObjID, + stackDetectRowID, + primaryDetection, +-- bestDetection, + dvoRegionID, + dataRelease, + gippDetectID, + gstackDetectID, + gstackMetaID, + HEX(ginfoFlag), + HEX(ginfoFlag2), + gnFrames +from StackObjectThin +limit 50 Index: trunk/ippToPsps/test/sascheck.v1.sql =================================================================== --- trunk/ippToPsps/test/sascheck.v1.sql (revision 37246) +++ trunk/ippToPsps/test/sascheck.v1.sql (revision 37246) @@ -0,0 +1,44 @@ +-- compare g mags across stack tables +select + gra, + gdec, + graErr, + gdecErr, + gPSFMag, + gKronMag, + gApMag, + gExpMag, + gSerMag, + gDeVMag, + gpetMag, + gPSFMagErr, + gKronMagErr, + gApMagErr, + gExpMagErr, + gSerMagErr, + gDeVMagErr, + gpetMagErr, + gpsfLikelihood, + gpsfQf, + gpsfQfPerfect, + gippDetectID + from StackObjectThin + -- left join StackObjectRaw using (gippDetectID) + -- left join StackModelFitExp using (gippDetectID) + -- left join StackModelFitDeV using (gippDetectID) + -- left join StackModelFitSer using (gippDetectID) + -- left join StackPetrosian using (gippDetectID) + -- left join StackApFlxFull using (gippDetectID) + -- left join StackApFlxFullC1 using (gippDetectID) + -- left join StackApFlxFullC2 using (gippDetectID) + join StackObjectRaw using (gippDetectID) + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + join StackPetrosian using (gippDetectID) + join StackApFlxFull using (gippDetectID) + join StackApFlxFullC1 using (gippDetectID) + join StackApFlxFullC2 using (gippDetectID) + -- where (gippDetectID = 189) + -- where (gpsfQfPerfect > 0.85) + limit 50 Index: trunk/ippToPsps/test/sascheck.v10.sql =================================================================== --- trunk/ippToPsps/test/sascheck.v10.sql (revision 37246) +++ trunk/ippToPsps/test/sascheck.v10.sql (revision 37246) @@ -0,0 +1,28 @@ +-- compare g fluxes across tables +select + gra, + gdec, + gPSFFlux, + gApFlux, + gPSFFluxErr, + gApFluxErr, + gApFillFac, + StackApFlx.gflxR3 as gflxR3_v1, + StackApFlxFull.gflxR3 as gflxR3_v2, + StackApFlx.gflxR4 as gflxR4_v1, + StackApFlxFull.gflxR4 as gflxR4_v2, + StackApFlx.gflxR5 as gflxR5_v1, + StackApFlxFull.gflxR5 as gflxR5_v2, + StackApFlx.gflxR3Err as gflxR3Err_v1, + StackApFlxFull.gflxR3Err as gflxR3Err_v2, + StackApFlx.gflxR4Err as gflxR4Err_v1, + StackApFlxFull.gflxR4Err as gflxR4Err_v2, + StackApFlx.gflxR5Err as gflxR5Err_v1, + StackApFlxFull.gflxR5Err as gflxR5Err_v2, + gippDetectID + from StackObjectThin + join StackObjectRaw using (gippDetectID) + join StackApFlx using (gippDetectID) + join StackApFlxFull using (gippDetectID) + where gPSFFlux > 1000 + limit 50 Index: trunk/ippToPsps/test/sascheck.v11.sql =================================================================== --- trunk/ippToPsps/test/sascheck.v11.sql (revision 37246) +++ trunk/ippToPsps/test/sascheck.v11.sql (revision 37246) @@ -0,0 +1,21 @@ +-- compare g fluxes across tables +select + gra, + gdec, + gPSFMag, + gPSFMagErr, + gpetRadius, + gpetRadiusErr, + gpetMag, + gpetMagErr, + gpetR50, + gpetR50Err, + gpetR90, + gpetR90Err, + gpetCf, + gippDetectID + from StackObjectThin + join StackObjectRaw using (gippDetectID) + join StackPetrosian using (gippDetectID) + where gPSFFlux > 1000 + limit 50 Index: trunk/ippToPsps/test/sascheck.v2.sql =================================================================== --- trunk/ippToPsps/test/sascheck.v2.sql (revision 37246) +++ trunk/ippToPsps/test/sascheck.v2.sql (revision 37246) @@ -0,0 +1,29 @@ +-- compare g raw PSF values +select + gippDetectID, + gxPos, + gyPos, + gxPosErr, + gyPosErr, + gpsfMajorFWHM, + gpsfMinorFWHM, + gpsfTheta, + gpsfCore, + gpsfLikelihood, + gpsfQf, + gpsfQfPerfect, + gpsfChiSq, + gmomentXX, + gmomentXY, + gmomentYY, + gmomentR1, + gmomentRH, + gApFillFac, + gKronRad, + gsky, + gskyErr, + gexpTime, + gzp + from StackObjectThin + join StackObjectRaw using (gippDetectID) + limit 80 Index: trunk/ippToPsps/test/sascheck.v3.sql =================================================================== --- trunk/ippToPsps/test/sascheck.v3.sql (revision 37246) +++ trunk/ippToPsps/test/sascheck.v3.sql (revision 37246) @@ -0,0 +1,32 @@ +-- compare g fluxes across tables +select + gra, + gdec, + gPSFFlux, + gKronFlux, + gApFlux, + gPSFFluxErr, + gKronFluxErr, + gApFluxErr, + gApFillFac, + gflxR1, + gflxR2, + gflxR3, + gflxR4, + gflxR5, + gflxR6, + gflxR7, + gflxR8, + gflxR9, + gflxR10, + gippDetectID + from StackObjectThin + join StackObjectRaw using (gippDetectID) + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + join StackPetrosian using (gippDetectID) + join StackApFlxFull using (gippDetectID) + join StackApFlxFullC1 using (gippDetectID) + join StackApFlxFullC2 using (gippDetectID) + limit 50 Index: trunk/ippToPsps/test/sascheck.v4.sql =================================================================== --- trunk/ippToPsps/test/sascheck.v4.sql (revision 37246) +++ trunk/ippToPsps/test/sascheck.v4.sql (revision 37246) @@ -0,0 +1,38 @@ +-- compare g fluxes across tables +select + gra, + gdec, + gPSFFlux, + gKronFlux, + gApFlux, + gflxR1, + gc1flxR1, + gc2flxR1, + gflxR2, + gc1flxR2, + gc2flxR2, + gflxR3, + gc1flxR3, + gc2flxR3, + gflxR4, + gc1flxR4, + gc2flxR4, + gflxR5, + gc1flxR5, + gc2flxR5, + gPSFFluxErr, + gKronFluxErr, + gApFluxErr, + gApFillFac, + gpsfQfPerfect, + gippDetectID + from StackObjectThin + join StackObjectRaw using (gippDetectID) + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + join StackPetrosian using (gippDetectID) + join StackApFlxFull using (gippDetectID) + join StackApFlxFullC1 using (gippDetectID) + join StackApFlxFullC2 using (gippDetectID) + limit 50 Index: trunk/ippToPsps/test/sascheck.v5.sql =================================================================== --- trunk/ippToPsps/test/sascheck.v5.sql (revision 37246) +++ trunk/ippToPsps/test/sascheck.v5.sql (revision 37246) @@ -0,0 +1,32 @@ +-- compare g mags across stack tables +select + gra, + gdec, + gExpMag, + gSerMag, + gDeVMag, + gExpRadius, + gSerRadius, + gDeVRadius, + gExpAb, + gSerAb, + gDeVAb, + gExpPhi, + gSerPhi, + gDeVPhi, + gippDetectID + from StackObjectThin + -- left join StackObjectRaw using (gippDetectID) + -- left join StackModelFitExp using (gippDetectID) + -- left join StackModelFitDeV using (gippDetectID) + -- left join StackModelFitSer using (gippDetectID) + -- left join StackPetrosian using (gippDetectID) + -- left join StackApFlxFull using (gippDetectID) + -- left join StackApFlxFullC1 using (gippDetectID) + -- left join StackApFlxFullC2 using (gippDetectID) + join StackObjectRaw using (gippDetectID) + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + -- where (gpsfQfPerfect > 0.85) + limit 50 Index: trunk/ippToPsps/test/sascheck.v6.sql =================================================================== --- trunk/ippToPsps/test/sascheck.v6.sql (revision 37246) +++ trunk/ippToPsps/test/sascheck.v6.sql (revision 37246) @@ -0,0 +1,25 @@ +-- compare g mags across stack tables +select + gra, + gdec, + gExpMag, + gExpMagErr, + gExpRadius, + gExpRadiusErr, + gExpAb, + gExpAbErr, + gExpPhi, + gExpPhiErr, + gExpRa, + gExpDec, + gExpRaErr, + gExpDecErr, + gExpCf, + gExpLikelihood, + gippDetectID + from StackObjectThin + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + -- where (gpsfQfPerfect > 0.85) + limit 50 Index: trunk/ippToPsps/test/sascheck.v7.sql =================================================================== --- trunk/ippToPsps/test/sascheck.v7.sql (revision 37246) +++ trunk/ippToPsps/test/sascheck.v7.sql (revision 37246) @@ -0,0 +1,25 @@ +-- compare g mags across stack tables +select + gra, + gdec, + gDeVMag, + gDeVMagErr, + gDeVRadius, + gDeVRadiusErr, + gDeVAb, + gDeVAbErr, + gDeVPhi, + gDeVPhiErr, + gDeVRa, + gDeVDec, + gDeVRaErr, + gDeVDecErr, + gDeVCf, + gDeVLikelihood, + gippDetectID + from StackObjectThin + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + -- where (gpsfQfPerfect > 0.85) + limit 50 Index: trunk/ippToPsps/test/sascheck.v8.sql =================================================================== --- trunk/ippToPsps/test/sascheck.v8.sql (revision 37246) +++ trunk/ippToPsps/test/sascheck.v8.sql (revision 37246) @@ -0,0 +1,27 @@ +-- compare g mags across stack tables +select + gra, + gdec, + gSerMag, + gSerMagErr, + gSerRadius, + gSerRadiusErr, + gSerAb, + gSerAbErr, + gSerNu, + gSerNuErr, + gSerPhi, + gSerPhiErr, + gSerRa, + gSerDec, + gSerRaErr, + gSerDecErr, + gSerCf, + gSerLikelihood, + gippDetectID + from StackObjectThin + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + -- where (gpsfQfPerfect > 0.85) + limit 50 Index: trunk/ippToPsps/test/sascheck.v9.sql =================================================================== --- trunk/ippToPsps/test/sascheck.v9.sql (revision 37246) +++ trunk/ippToPsps/test/sascheck.v9.sql (revision 37246) @@ -0,0 +1,29 @@ +-- compare g fluxes across tables +select + gra, + gdec, + gxPos, + gyPos, + gPSFFlux, + gApFlux, + gPSFFluxErr, + gApFluxErr, + gApFillFac, + gflxR3, + gflxR4, + gflxR5, + gflxR3Err, + gflxR4Err, + gflxR5Err, + gflxR3Std, + gflxR4Std, + gflxR5Std, + gflxR3Fill, + gflxR4Fill, + gflxR5Fill, + gippDetectID + from StackObjectThin + join StackObjectRaw using (gippDetectID) + join StackApFlx using (gippDetectID) + where gPSFFlux > 1000 + limit 50 Index: trunk/ippToPsps/test/sastest.sh =================================================================== --- trunk/ippToPsps/test/sastest.sh (revision 35753) +++ trunk/ippToPsps/test/sastest.sh (revision 37246) @@ -1,9 +1,5 @@ #!/bin/csh -f -set stackqueuename = 0 -if ($#argv > 0) then - set stackqueuename = $1 - echo "stackqueue: $stackqueuename" -endif +chmod +x mksasdata.dvo mkgpc1.sh # a bit crude : get the datadir path by interpolation of the PATH element for psconfig @@ -19,18 +15,136 @@ # mkgpc1.sh user localhost eam eam -set mkgpc1 = 1 -set initdb = 1 -set mkdummy = 1 -set initscratch = 1 -set initbatch = 1 -set camqueue = 1 -set cambatch = 1 -set stackqueue = 0 -set stackbatch = 0 - -set OUTDIR = "sas12" +set mkgpc1 = 1 +set initdb = 1 +set mkdummy = 1 +set initscratch = 1 +set initbatch = 1 +set camqueue = 1 +set cambatch = 1 +set stackqueue = 1 +set stackbatch = 1 +set objectqueue = 1 +set objectbatch = 1 + +set targ = ($argv) +set args = "" + +while ($#targ) + switch ("$targ[1]") + case -h + case -help + echo "USAGE: fulltest.sh [-XX] [+XX]" + echo " -XX : turn off features" + echo " +XX : turn on features" + echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, object" + exit 2 + case -all + set mkgpc1 = 0 + set initdb = 0 + set initscratch = 0 + set initbatch = 0 + set camqueue = 0 + set cambatch = 0 + set stackqueue = 0 + set stackbatch = 0 + set objectqueue = 0 + set objectbatch = 0 + breaksw; + case -mkgpc1 + set mkgpc1 = 0 + breaksw; + case -initdb + set initdb = 0 + breaksw; + case -initscratch + set initscratch = 0 + breaksw; + case -initbatch + set initbatch = 0 + breaksw; + case -camera + set camqueue = 0 + set cambatch = 0 + breaksw; + case -stack + set stackqueue = 0 + set stackbatch = 0 + breaksw; + case -object + set objectqueue = 0 + set objectbatch = 0 + breaksw; + default: + set args=($args $targ[1]); + breaksw; + endsw + shift targ +end + +set targ = ($args) +set args = "" + +while ($#targ) + switch ("$targ[1]") + case +all + set mkgpc1 = 1 + set initdb = 1 + set initscratch = 1 + set initbatch = 1 + set camqueue = 1 + set cambatch = 1 + set stackqueue = 1 + set stackbatch = 1 + set objectqueue = 1 + set objectbatch = 1 + breaksw; + case +mkgpc1 + set mkgpc1 = 1 + breaksw; + case +initdb + set initdb = 1 + breaksw; + case +initscratch + set initscratch = 1 + breaksw; + case +initbatch + set initbatch = 1 + breaksw; + case "+camera" + set camqueue = 1 + set cambatch = 1 + breaksw; + case +stack + set stackqueue = 1 + set stackbatch = 1 + breaksw; + case +object + set objectqueue = 1 + set objectbatch = 1 + breaksw; + default: + set args=($args $targ[1]); + breaksw; + endsw + shift targ +end + +if ($args[1] != "") then + echo "USAGE: sastest.sh [-XX] [+XX]" + echo " -XX : turn off features" + echo " +XX : turn on features" + echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, object" + exit 2 +endif + +set stackqueuename = 0 +set objectqueuename = 0 + +set OUTDIR = "sasdata" set queuename = "test" if ($mkgpc1) then + echo "" + echo " ---- mksasdata.dvo : creating the dvo & gpc1 database entries ----" mksasdata.dvo endif @@ -38,4 +152,6 @@ # create the ipptopsps database if ($initdb) then + echo "" + echo " ---- init ipptopsps_test ----" # does the database already exist? @@ -43,5 +159,4 @@ mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_test" mysql -h localhost -u dvo -pdvo ipptopsps_test < $IPPTOPSPS_DATA/ippToPspsDbSchema.sql - endif @@ -52,89 +167,219 @@ mysql -h localhost -u dvo -pdvo -e "drop database ipptopsps_scratch" mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_scratch" - ippjython setupScratchDb.py -test + ippjython setupScratchDb.py -test < /dev/null + if ($status) exit +endif + +# this is poor : we need to queue something in order to generate an init batch +# this is because a skychunk carries the info about the datastore used to publish +# the batch +if ($initbatch) then + echo "" + echo " ---- make skychunk for IN batches ----" + + echo "initbatch " > initbatch.dat # skychunk name + echo "test_ds " >> initbatch.dat # datastore_product + echo "IPP_PSPS " >> initbatch.dat # datastore_type + echo "0 " >> initbatch.dat # datastore_publish + echo "none " >> initbatch.dat # dvo_label + echo "none " >> initbatch.dat # dvo_location + echo "0 " >> initbatch.dat # min_ra + echo "0 " >> initbatch.dat # max_ra + echo "0 " >> initbatch.dat # min_dec + echo "0 " >> initbatch.dat # max_dec + echo "2 " >> initbatch.dat # box_size + echo "none " >> initbatch.dat # base_path + echo "0 " >> initbatch.dat # data_release + echo "0 " >> initbatch.dat # delete_local + echo "1 " >> initbatch.dat # delete_datastore + echo "1 " >> initbatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> initbatch.dat # epoch + echo "3PI " >> initbatch.dat # survey + echo "3PI " >> initbatch.dat # psps_survey + echo "0 " >> initbatch.dat # queue_P2 + echo "0 " >> initbatch.dat # queue_ST + echo "0 " >> initbatch.dat # queue_OB + echo "0 " >> initbatch.dat # queue_DF + echo "0 " >> initbatch.dat # queue_DO + echo "0 " >> initbatch.dat # queue_FW + echo "0 " >> initbatch.dat # queue_FO + echo "1 " >> initbatch.dat # active + echo "0 " >> initbatch.dat # parallel + echo "use_new " >> initbatch.dat # P2_smf_version + echo "any " >> initbatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> initbatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> initbatch.dat # trange_end + cat initbatch.dat | ippjython queue.py -test edit + if ($status) exit + + echo "" + echo " ---- create init batch ----" + + rm -rf $OUTDIR/IN + ippjython loader.py -test initbatch init < /dev/null + if ($status) exit endif # this code is sensitive to a previously existing database? if ($camqueue) then - ippjython queue.py -test edit < cambatch.dat # skychunk name + echo "test_ds " >> cambatch.dat # datastore_product + echo "IPP_PSPS " >> cambatch.dat # datastore_type + echo "0 " >> cambatch.dat # datastore_publish + echo "catdir.sas.cam " >> cambatch.dat # dvo_label + echo "$OUTDIR/catdir.sas.cam" >> cambatch.dat # dvo_location + echo "330 " >> cambatch.dat # min_ra + echo "340 " >> cambatch.dat # max_ra + echo "-10 " >> cambatch.dat # min_dec + echo "0 " >> cambatch.dat # max_dec + echo "1 " >> cambatch.dat # box_size + echo "$OUTDIR " >> cambatch.dat # base_path + echo "0 " >> cambatch.dat # data_release + echo "0 " >> cambatch.dat # delete_local + echo "1 " >> cambatch.dat # delete_datastore + echo "1 " >> cambatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> cambatch.dat # epoch + echo "3PI " >> cambatch.dat # survey + echo "3PI " >> cambatch.dat # psps_survey + echo "1 " >> cambatch.dat # queue_P2 + echo "0 " >> cambatch.dat # queue_ST + echo "0 " >> cambatch.dat # queue_OB + echo "0 " >> cambatch.dat # queue_DF + echo "0 " >> cambatch.dat # queue_DO + echo "0 " >> cambatch.dat # queue_FW + echo "0 " >> cambatch.dat # queue_FO + echo "1 " >> cambatch.dat # active + echo "0 " >> cambatch.dat # parallel + echo "use_new " >> cambatch.dat # P2_smf_version + echo "any " >> cambatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> cambatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> cambatch.dat # trange_end + cat cambatch.dat | ippjython queue.py -test edit + if ($status) exit endif if ($cambatch) then - ippjython loader.py -test $queuename once + echo "" + echo " ---- load P2 batches ----" + + rm -rf $OUTDIR/P2 + ippjython loader.py -test $queuename once < /dev/null + if ($status) exit endif # this code is sensitive to a previously existing database? if ($stackqueue) then + echo "" + echo " ---- queue ST batches ----" + if ($stackqueuename == 0) then set stackqueuename = $queuename\_stk endif - ippjython queue.py -test edit < stackbatch.dat # skychunk name + echo "test_ds " >> stackbatch.dat # datastore_product + echo "IPP_PSPS " >> stackbatch.dat # datastore_type + echo "0 " >> stackbatch.dat # datastore_publish + echo "catdir.sas.cam " >> stackbatch.dat # dvo_label + echo "$OUTDIR/catdir.sas.cam" >> stackbatch.dat # dvo_location + echo "332 " >> stackbatch.dat # min_ra + echo "335 " >> stackbatch.dat # max_ra + echo "-6 " >> stackbatch.dat # min_dec + echo "-4 " >> stackbatch.dat # max_dec + echo "1 " >> stackbatch.dat # box_size + echo "$OUTDIR " >> stackbatch.dat # base_path + echo "0 " >> stackbatch.dat # data_release + echo "0 " >> stackbatch.dat # delete_local + echo "1 " >> stackbatch.dat # delete_datastore + echo "1 " >> stackbatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> stackbatch.dat # epoch + echo "3PI " >> stackbatch.dat # survey + echo "3PI " >> stackbatch.dat # psps_survey + echo "0 " >> stackbatch.dat # queue_P2 + echo "1 " >> stackbatch.dat # queue_ST + echo "0 " >> stackbatch.dat # queue_OB + echo "0 " >> stackbatch.dat # queue_DF + echo "0 " >> stackbatch.dat # queue_DO + echo "0 " >> stackbatch.dat # queue_FW + echo "0 " >> stackbatch.dat # queue_FO + echo "1 " >> stackbatch.dat # active + echo "0 " >> stackbatch.dat # parallel + echo "use_new " >> stackbatch.dat # P2_smf_version + echo "any " >> stackbatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> stackbatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> stackbatch.dat # trange_end + cat stackbatch.dat | ippjython queue.py -test edit + if ($status) exit endif if ($stackbatch) then + echo "" + echo " ---- load ST batches ----" + if ($stackqueuename == 0) then set stackqueuename = $queuename\_stk endif - ippjython loader.py -test $stackqueuename once -endif - + rm -rf $OUTDIR/ST + ippjython loader.py -test $stackqueuename once < /dev/null + if ($status) exit +endif + +# this code is sensitive to a previously existing database? +if ($objectqueue) then + echo "" + echo " ---- queue OB batches ----" + + if ($objectqueuename == 0) then + set objectqueuename = $queuename\_obj + endif + echo "$objectqueuename " > objectbatch.dat # skychunk name + echo "test_ds " >> objectbatch.dat # datastore_product + echo "IPP_PSPS " >> objectbatch.dat # datastore_type + echo "0 " >> objectbatch.dat # datastore_publish + echo "catdir.sas.cam " >> objectbatch.dat # dvo_label + echo "$OUTDIR/catdir.sas.cam " >> objectbatch.dat # dvo_location + echo "330 " >> objectbatch.dat # min_ra + echo "340 " >> objectbatch.dat # max_ra + echo "-10 " >> objectbatch.dat # min_dec + echo "0 " >> objectbatch.dat # max_dec + echo "2 " >> objectbatch.dat # box_size + echo "$OUTDIR " >> objectbatch.dat # base_path + echo "0 " >> objectbatch.dat # data_release + echo "0 " >> objectbatch.dat # delete_local + echo "1 " >> objectbatch.dat # delete_datastore + echo "1 " >> objectbatch.dat # delete_dxlayer + echo "2012-12-10 00:00:00.0 " >> objectbatch.dat # epoch + echo "3PI " >> objectbatch.dat # survey + echo "3PI " >> objectbatch.dat # psps_survey + echo "0 " >> objectbatch.dat # queue_P2 + echo "0 " >> objectbatch.dat # queue_ST + echo "1 " >> objectbatch.dat # queue_OB + echo "0 " >> objectbatch.dat # queue_DF + echo "0 " >> objectbatch.dat # queue_DO + echo "0 " >> objectbatch.dat # queue_FW + echo "0 " >> objectbatch.dat # queue_FO + echo "1 " >> objectbatch.dat # active + echo "0 " >> objectbatch.dat # parallel + echo "use_new " >> objectbatch.dat # P2_smf_version + echo "any " >> objectbatch.dat # ST_cmf_version + echo "2000-01-01 00:00:00.0 " >> objectbatch.dat # trange_start + echo "2020-01-01 00:00:00.0 " >> objectbatch.dat # trange_end + cat objectbatch.dat | ippjython queue.py -test edit once + if ($status) exit +endif + +if ($objectbatch) then + echo "" + echo " ---- load OB batches ----" + + if ($objectqueuename == 0) then + set objectqueuename = $queuename\_obj + endif + rm -rf $OUTDIR/OB + + ippjython loader.py -test $objectqueuename once < /dev/null + if ($status) exit +endif +