Changeset 38888
- Timestamp:
- Oct 20, 2015, 2:07:03 PM (11 years ago)
- Location:
- trunk/ippToPsps/test
- Files:
-
- 1 added
- 3 edited
-
fulltest.sh (modified) (8 diffs)
-
gpc1schema.sql (added)
-
mkgpc1.sh (modified) (4 diffs)
-
mkgpc1data.dvo (modified) (11 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippToPsps/test/fulltest.sh
r38840 r38888 23 23 set stackqueue = 1 24 24 set stackbatch = 1 25 set forcequeue = 0 26 set forcebatch = 0 25 27 set objectqueue = 1 26 28 set objectbatch = 1 … … 36 38 echo " -XX : turn off features" 37 39 echo " +XX : turn on features" 38 echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, object"40 echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, forcedwarp, object" 39 41 exit 2 40 42 case -all … … 47 49 set stackqueue = 0 48 50 set stackbatch = 0 51 set forcequeue = 0 52 set forcebatch = 0 49 53 set objectqueue = 0 50 54 set objectbatch = 0 … … 69 73 set stackqueue = 0 70 74 set stackbatch = 0 75 breaksw; 76 case -forcedwarp 77 set forcequeue = 0 78 set forcebatch = 0 71 79 breaksw; 72 80 case -object … … 95 103 set stackqueue = 1 96 104 set stackbatch = 1 105 set forcequeue = 1 106 set forcebatch = 1 97 107 set objectqueue = 1 98 108 set objectbatch = 1 … … 117 127 set stackqueue = 1 118 128 set stackbatch = 1 129 breaksw; 130 case +forcedwarp 131 set forcequeue = 1 132 set forcebatch = 1 119 133 breaksw; 120 134 case +object … … 133 147 echo " -XX : turn off features" 134 148 echo " +XX : turn on features" 135 echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, object"149 echo " features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, forcedwarp, object" 136 150 exit 2 137 151 endif … … 334 348 335 349 # this code is sensitive to a previously existing database? 350 if ($forcequeue) then 351 echo "" 352 echo " ---- queue FW batches ----" 353 354 echo "$queuename " > forcebatch.dat # skychunk name 355 echo "test_ds " >> forcebatch.dat # datastore_product 356 echo "IPP_PSPS " >> forcebatch.dat # datastore_type 357 echo "0 " >> forcebatch.dat # datastore_publish 358 echo "catdir.wrp " >> forcebatch.dat # dvo_label 359 echo "$OUTDIR/catdir.wrp " >> forcebatch.dat # dvo_location 360 echo "9 " >> forcebatch.dat # min_ra 361 echo "11 " >> forcebatch.dat # max_ra 362 echo "19 " >> forcebatch.dat # min_dec 363 echo "21 " >> forcebatch.dat # max_dec 364 echo "2 " >> forcebatch.dat # box_size 365 echo "$OUTDIR " >> forcebatch.dat # base_path 366 echo "0 " >> forcebatch.dat # data_release 367 echo "0 " >> forcebatch.dat # delete_local 368 echo "1 " >> forcebatch.dat # delete_datastore 369 echo "1 " >> forcebatch.dat # delete_dxlayer 370 echo "2012-12-10 00:00:00.0 " >> forcebatch.dat # epoch 371 echo "3PI " >> forcebatch.dat # survey 372 echo "3PI " >> forcebatch.dat # psps_survey 373 echo "0 " >> forcebatch.dat # queue_P2 374 echo "0 " >> forcebatch.dat # queue_ST 375 echo "0 " >> forcebatch.dat # queue_OB 376 echo "0 " >> forcebatch.dat # queue_DF 377 echo "0 " >> forcebatch.dat # queue_DO 378 echo "1 " >> forcebatch.dat # queue_FW 379 echo "0 " >> forcebatch.dat # queue_FO 380 echo "1 " >> forcebatch.dat # active 381 echo "0 " >> forcebatch.dat # parallel 382 echo "use_new " >> forcebatch.dat # P2_smf_version 383 echo "any " >> forcebatch.dat # ST_cmf_version 384 echo "2000-01-01 00:00:00.0 " >> forcebatch.dat # trange_start 385 echo "2020-01-01 00:00:00.0 " >> forcebatch.dat # trange_end 386 ippjython queue.py -test edit -simtest < forcebatch.dat 387 if ($status) exit 1 388 endif 389 390 if ($forcebatch) then 391 echo "" 392 echo " ---- load FW batches ----" 393 394 rm -rf $OUTDIR/FW 395 ippjython loader.py -test $queuename once -simtest < /dev/null 396 if ($status) exit 397 endif 398 399 # this code is sensitive to a previously existing database? 336 400 if ($objectqueue) then 337 401 echo "" -
trunk/ippToPsps/test/mkgpc1.sh
r37246 r38888 11 11 set dbpass = $4 12 12 13 mysql -h $dbhost -u root -p <<EOF 14 grant all on *.* to $dbuser@"$dbhost" identified by '$dbpass'; 15 EOF 16 exit 0; 13 mysql -h $dbhost -u root -p -e "grant all on *.* to $dbuser@"$dbhost" identified by '$dbpass'" 14 exit 0; 17 15 endif 18 16 … … 29 27 30 28 # create a database with completely minimal tables needed to test 31 mysql -h $dbhost -u $dbuser $dbpass <<EOF 32 CREATE DATABASE $dbname; 33 USE $dbname; 34 CREATE TABLE ippToPspsFake ( 35 my_id INT 36 ); 37 CREATE TABLE skycell ( 38 radeg REAL, 39 decdeg REAL, 40 skycell_id CHAR(64) 41 ); 42 CREATE TABLE rawExp ( 43 exp_id INT, 44 exp_name CHAR(64), 45 exp_time REAL, 46 bg REAL, 47 bg_stdev REAL, 48 ra FLOAT, 49 decl FLOAT, 50 dateobs DATETIME 51 ); 52 CREATE TABLE chipRun ( 53 exp_id INT, 54 chip_id INT 55 ); 56 CREATE TABLE camRun ( 57 chip_id INT, 58 cam_id INT, 59 dist_group CHAR(64), 60 software_ver CHAR(16) 61 ); 62 CREATE TABLE camProcessedExp ( 63 path_base CHAR(255), 64 cam_id INT, 65 fault INT 66 ); 67 CREATE TABLE fakeRun ( 68 fake_id INT, 69 cam_id INT 70 ); 71 CREATE TABLE warpRun ( 72 warp_id INT, 73 fake_id INT 74 ); 75 CREATE TABLE warpSkyCellMap ( 76 warp_id INT, 77 skycell_id CHAR(64), 78 class_id CHAR(64) 79 ); 80 CREATE TABLE stackRun ( 81 stack_id INT, 82 skycell_id CHAR(64), 83 filter CHAR(64), 84 software_ver CHAR(16) 85 ); 86 CREATE TABLE stackInputSkyfile ( 87 stack_id INT, 88 warp_id INT 89 ); 90 CREATE TABLE stackSumSkyfile ( 91 stack_id INT, 92 mjd_obs DOUBLE 93 ); 94 CREATE TABLE skycalRun ( 95 sky_id INT, 96 skycal_id INT, 97 stack_id INT 98 ); 99 CREATE TABLE skycalResult ( 100 path_base CHAR(255), 101 skycal_id INT 102 ); 103 CREATE TABLE addRun ( 104 add_id INT, 105 stage CHAR(64), 106 state CHAR(64), 107 stage_id INT, 108 minidvodb_name CHAR(64) 109 ); 110 CREATE TABLE addProcessedExp ( 111 add_id INT, 112 fault INT 113 ); 114 CREATE TABLE minidvodbRun ( 115 minidvodb_name CHAR(64), 116 minidvodb_id INT, 117 state CHAR(64) 118 ); 119 CREATE TABLE minidvodbProcessed ( 120 minidvodb_id INT, 121 fault INT 122 ); 123 CREATE TABLE mergedvodbRun ( 124 minidvodb_id INT, 125 merge_id INT, 126 state CHAR(64), 127 mergedvodb CHAR(64) 128 ); 129 CREATE TABLE mergedvodbProcessed ( 130 merge_id INT, 131 fault INT 132 ); 133 EOF 134 exit 0; 29 mysql -h $dbhost -u $dbuser $dbpass -e "CREATE DATABASE $dbname" 30 mysql -h $dbhost -u $dbuser $dbpass $dbname < gpc1schema.sql 31 exit 0; 135 32 endif 136 33 … … 146 43 endif 147 44 148 mysql -h $dbhost -u $dbuser $dbpass <<EOF 149 USE $dbname; 150 describe ippToPspsFake; 151 EOF 45 (mysql -B -h $dbhost -u $dbuser $dbpass -e "show databases" | grep ^$dbname\$) >& /dev/null 46 if ($status) then 47 echo "database $dbname does not yet exist" 48 exit 0 49 endif 50 51 # mysql -h $dbhost -u $dbuser $dbpass $dbname -e "show tables" 52 mysql -h $dbhost -u $dbuser $dbpass $dbname -e "describe ippToPspsFake" >& /dev/null 152 53 if ($status) then 153 54 echo "database does not contain ippToPspsFake, not deleting" … … 155 56 endif 156 57 157 set ntable = ` echo "USE $dbname; show tables;" | mysql -h $dbhost -u $dbuser $dbpass| wc -l`58 set ntable = `mysql -B -h $dbhost -u $dbuser $dbpass $dbname -e "show tables" | wc -l` 158 59 echo ntable: $ntable 159 if ($ntable != 2 1) then160 echo "expected 2 1 lines (20tables), but got $ntable, not deleting"60 if ($ntable != 25) then 61 echo "expected 25 lines (24 tables), but got $ntable, not deleting" 161 62 exit 2 162 63 endif 163 64 164 65 echo "database is valid, deleting" 165 mysql -h $dbhost -u $dbuser $dbpass <<EOF 166 drop database $dbname; 167 EOF 168 169 exit 0; 66 mysql -h $dbhost -u $dbuser $dbpass -e "drop database $dbname" 67 exit 0; 170 68 endif 171 69 -
trunk/ippToPsps/test/mkgpc1data.dvo
r38850 r38888 12 12 # I'm setting the base filename without the .smf and add it in mkcatdir.* 13 13 14 # some global value for ease of access: 15 $SKYCELL = "skycell.1133.081" 16 $TESS_ID = "RINGS.V3" 17 14 18 # imagedata describes the full fake exposure set. the fields are 15 19 list imagedata 16 # exp_id stk_id filebase M_off date time 17 1 1 $OUTDIR/test.01 0.000 2010/01/01 01:00:00 18 2 1 $OUTDIR/test.02 -0.025 2010/01/01 02:00:00 19 3 1 $OUTDIR/test.03 0.025 2010/01/01 03:00:00 20 4 1 $OUTDIR/test.04 0.010 2010/01/01 04:00:00 21 5 2 $OUTDIR/test.05 0.000 2010/01/01 05:00:00 22 6 2 $OUTDIR/test.06 -0.025 2010/01/01 06:00:00 23 7 2 $OUTDIR/test.07 0.025 2010/01/01 07:00:00 24 8 2 $OUTDIR/test.08 0.010 2010/01/01 08:00:00 25 9 3 $OUTDIR/test.09 0.000 2010/01/01 09:00:00 26 10 3 $OUTDIR/test.10 -0.025 2010/01/01 10:00:00 27 11 3 $OUTDIR/test.11 0.025 2010/01/01 11:00:00 28 12 3 $OUTDIR/test.12 0.010 2010/01/01 12:00:00 29 end 30 31 list warpdata 32 1 1 $OUTDIR/test.01 0.000 2010/01/01 01:00:00 33 2 1 $OUTDIR/test.02 -0.025 2010/01/01 02:00:00 34 3 1 $OUTDIR/test.03 0.025 2010/01/01 03:00:00 35 4 1 $OUTDIR/test.04 0.010 2010/01/01 04:00:00 36 5 2 $OUTDIR/test.05 0.000 2010/01/01 05:00:00 37 6 2 $OUTDIR/test.06 -0.025 2010/01/01 06:00:00 38 7 2 $OUTDIR/test.07 0.025 2010/01/01 07:00:00 39 8 2 $OUTDIR/test.08 0.010 2010/01/01 08:00:00 40 9 3 $OUTDIR/test.09 0.000 2010/01/01 09:00:00 41 10 3 $OUTDIR/test.10 -0.025 2010/01/01 10:00:00 42 11 3 $OUTDIR/test.11 0.025 2010/01/01 11:00:00 43 12 3 $OUTDIR/test.12 0.010 2010/01/01 12:00:00 20 # exp_id stk_id filebase M_off date time warpfile 21 1 1 $OUTDIR/test.01 0.000 2010/01/01 01:00:00 $OUTDIR/warp.01 22 2 1 $OUTDIR/test.02 -0.025 2010/01/01 02:00:00 $OUTDIR/warp.02 23 3 1 $OUTDIR/test.03 0.025 2010/01/01 03:00:00 $OUTDIR/warp.03 24 4 1 $OUTDIR/test.04 0.010 2010/01/01 04:00:00 $OUTDIR/warp.04 25 5 2 $OUTDIR/test.05 0.000 2010/01/01 05:00:00 $OUTDIR/warp.05 26 6 2 $OUTDIR/test.06 -0.025 2010/01/01 06:00:00 $OUTDIR/warp.06 27 7 2 $OUTDIR/test.07 0.025 2010/01/01 07:00:00 $OUTDIR/warp.07 28 8 2 $OUTDIR/test.08 0.010 2010/01/01 08:00:00 $OUTDIR/warp.08 29 9 3 $OUTDIR/test.09 0.000 2010/01/01 09:00:00 $OUTDIR/warp.09 30 10 3 $OUTDIR/test.10 -0.025 2010/01/01 10:00:00 $OUTDIR/warp.10 31 11 3 $OUTDIR/test.11 0.025 2010/01/01 11:00:00 $OUTDIR/warp.11 32 12 3 $OUTDIR/test.12 0.010 2010/01/01 12:00:00 $OUTDIR/warp.12 44 33 end 45 34 … … 55 44 mkcatdir.cam PS1_V5 PS1_V5 56 45 mkcatdir.stk PS1_V5 PS1_V5 46 mkcatdir.wrp PS1_V5_Lensing PS1_V5 57 47 insert.stack.set 58 48 end … … 68 58 $DEC = 20.0 69 59 70 break -auto off71 60 exec mkgpc1.sh delete $dbhost $dbname $dbuser $dbpass 72 break -auto on73 61 exec mkgpc1.sh create $dbhost $dbname $dbuser $dbpass 74 62 … … 77 65 # system-wide info 78 66 dbinsert ippToPspsFake (my_id) values (2) 79 dbinsert skycell (radeg, decdeg, skycell_id) values ($RA, $DEC, "skycell.0000.000")67 dbinsert skycell (radeg, decdeg, skycell_id) values ($RA, $DEC, $SKYCELL) 80 68 end 81 69 … … 163 151 164 152 mkinput $offset $rawfile 165 echo mkcmf -photcode SIMTEST.r.Chip -no-noise -append $rawfile $cmffile -date $myDATE -time $myTIME - radec $RA $DEC -type $1 -imageID $ID -sourceID 0166 exec mkcmf -photcode SIMTEST.r.Chip -no-noise -append $rawfile $cmffile -date $myDATE -time $myTIME - radec $RA $DEC -type $1 -imageID $ID -sourceID 0153 echo mkcmf -photcode SIMTEST.r.Chip -no-noise -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 154 exec mkcmf -photcode SIMTEST.r.Chip -no-noise -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 167 155 168 156 if ($i == 3) … … 238 226 239 227 mkinput $offset $rawfile 240 echo mkcmf -stack -photcode SIMTEST.r.SkyChip $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 - radec $RA $DEC -type $1 -imageID $ID -sourceID 1241 exec mkcmf -stack -photcode SIMTEST.r.SkyChip $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 - radec $RA $DEC -type $1 -imageID $ID -sourceID 1228 echo mkcmf -stack -photcode SIMTEST.r.SkyChip $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $ID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL 229 exec mkcmf -stack -photcode SIMTEST.r.SkyChip $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $ID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL 242 230 243 231 # add some required header keywords: 244 232 exec echo "HIERARCH FPA.ZP = 25. / Magnitude zero point" > tmp.hdr 245 exec echo "TESS_ID = 'RINGS.V3'" >> tmp.hdr246 exec echo "SKYCELL = 'skycell.1133.081'" >> tmp.hdr247 233 exec fits_insert $cmffile tmp.hdr 248 234 … … 273 259 end 274 260 261 # create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp 262 macro mkcatdir.wrp 263 if ($0 != 3) 264 echo "mkcatdir.wrp (cmftype) (dvotype)" 265 break 266 end 267 268 # tapPLAN 4 269 270 mkdir $OUTDIR 271 272 local i catdir RA DEC ID rawfile cmffile offset 273 $catdir = $OUTDIR/catdir.wrp 274 exec rm -rf $catdir 275 276 $RA = 10.0 277 $DEC = 20.0 278 279 $TIMEFORMAT = mjd 280 $TIMEREF = 2001/01/01,00:00:00 281 282 for i 0 $imagedata:n 283 list word -split $imagedata:$i 284 $ID = $word:0 285 $rawfile = $word:2\.txt; # use the cam-stage rawfile 286 $cmffile = $word:6\.smf 287 $offset = $word:3 288 289 sprintf myDATE "%s" $word:4 290 sprintf myTIME "%s" $word:5 291 echo ctimes -abs $myDATE,$myTIME -var mjd 292 ctimes -abs $myDATE,$myTIME -var mjd 293 294 ## this is not generating photcodes recognized as stack by relphot 295 296 mkinput $offset $rawfile 297 echo mkcmf -forcedwarp -photcode SIMTEST.r.ForcedWarp $rawfile $cmffile -extroot SkyChip -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL 298 exec mkcmf -forcedwarp -photcode SIMTEST.r.ForcedWarp $rawfile $cmffile -extroot SkyChip -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL 299 300 echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass 301 exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass 302 end 303 304 echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update 305 exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update 306 307 if ($PARALLEL) 308 $hostname = `hostname` 309 exec rm -f $catdir/HostTable.dat 310 output $catdir/HostTable.dat 311 echo "# ID Hostname Catdir" 312 echo " 1 $hostname $catdir.p1" 313 echo " 2 $hostname $catdir.p2" 314 echo " 3 $hostname $catdir.p3" 315 output stdout 316 exec dvodist -out $catdir 317 end 318 319 # for i 0 $offset:n 320 # tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i" 321 # end 322 323 # tapDONE 324 end 325 275 326 macro insert.exp 276 327 if ($0 != 3) … … 296 347 dbinsert camProcessedExp (cam_id, path_base, fault) values ($myID, "$filename", 0) 297 348 dbinsert fakeRun (fake_id, cam_id) values ($myID, $myID) 298 dbinsert warpRun (warp_id, fake_id) values ($myID, $myID) 299 dbinsert warpSkyCellMap (warp_id, skycell_id, class_id) values ($myID, "skycell.0000.000", "XY00") 349 dbinsert warpRun (warp_id, fake_id, tess_id) values ($myID, $myID, $TESS_ID) 350 dbinsert warpSkyCellMap (warp_id, skycell_id, class_id) values ($myID, $SKYCELL, "XY00") 351 dbinsert warpImfile (warp_id, skycell_id, warp_skyfile_id) values ($myID, $SKYCELL, $myID) 300 352 301 353 dbinsert addRun (add_id, stage, state, stage_id, minidvodb_name) values ($myID, "cam", "full", $myID, "catdir.cam") 302 354 dbinsert addProcessedExp (add_id, fault) values ($myID, 0) 355 end 356 357 macro insert.wrp 358 if ($0 != 4) 359 echo "USAGE: insert.wrp (warpID) (stackID) (filename)" 360 break 361 end 362 363 local wrpID stkID filename 364 $wrpID = $1 365 $stkID = $2 366 $filename = $3 367 368 # a single exposure 369 echo $myDATE 370 echo $myTIME 371 372 dbinsert addRun (add_id, stage, state, stage_id, stage_extra1, minidvodb_name) values ($wrpID, "fullforce", "full", $stkID, $wrpID, "catdir.wrp") 373 374 # we have a single fullForceRun for each stack, so we can match fullForceRun ids to stackRun ids 375 dbinsert fullForceInput (ff_id, warp_id) values ($stkID, $wrpID) 376 dbinsert fullForceResult (ff_id, warp_id, software_ver, path_base) values ($stkID, $wrpID, "38000M", "$filename") 303 377 end 304 378 … … 310 384 end 311 385 312 local i stkID expID expfile stkfile386 local i stkID expID expfile wrpfile stkfile 313 387 314 388 $stkID = $1 315 389 $stkfile = $2 316 390 317 dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "skycell.0000.000", "g.00000", "37500") 391 dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, $SKYCELL, "g.00000", "37500") 392 393 # we have a single fullForceRun for each stack, so we can match fullForceRun ids to stackRun ids 394 dbinsert fullForceRun (ff_id, skycal_id) values ($stkID, $stkID) 318 395 319 396 $TIMEFORMAT = mjd … … 329 406 330 407 $expID = $tmp:0 331 $expfile = $tmp:2332 408 sprintf myDATE "%s" $tmp:4 333 409 sprintf myTIME "%s" $tmp:5 410 411 $expfile = $tmp:2 334 412 insert.exp $expID $expfile 413 414 $wrpfile = $tmp:6 415 insert.wrp $expID $stkID $wrpfile 416 335 417 dbinsert stackInputSkyfile (stack_id, warp_id) values ($stkID, $expID) 336 418 … … 376 458 dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (2, 2, "full", "catdir.stk") 377 459 dbinsert mergedvodbProcessed (merge_id, fault) values (2, 0) 460 461 # catdirs for the forced warps 462 dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.wrp", 3, "merged") 463 dbinsert minidvodbProcessed (minidvodb_id, fault) values (3, 0) 464 465 dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (3, 3, "full", "catdir.wrp") 466 dbinsert mergedvodbProcessed (merge_id, fault) values (3, 0) 378 467 end 379 468
Note:
See TracChangeset
for help on using the changeset viewer.
