Index: /branches/eam_branches/ipp-pstamp-20260421/pstamp/test/pstamptest.proc.sh =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/pstamp/test/pstamptest.proc.sh (revision 43092) +++ /branches/eam_branches/ipp-pstamp-20260421/pstamp/test/pstamptest.proc.sh (revision 43092) @@ -0,0 +1,69 @@ +#!/bin/csh + +echo "" + +set VERBOSE = +set VERBOSE = --verbose + +set STAGE1 = 1 +set STAGE2 = 1 +set STAGE3 = 1 + +if ($STAGE1) then + set REQ_ID = `pstamptool -pendingreq -dbname pstamp | grep req_id | tail -n 1 | awk '{print $3}'` + if ("$REQ_ID" == "") then + echo "No pending requests" + exit 2 + endif + + set URI = `pstamptool -pendingreq -dbname pstamp | grep uri | tail -n 1 | awk '{print $3}'` + set PRODUCT = `pstamptool -pendingreq -dbname pstamp | grep outProduct | grep -v ds_out | tail -n 1 | awk '{print $3}'` + set OUTDIR = `pstamptool -pendingreq -dbname pstamp | grep outdir | tail -n 1 | awk '{print $3}'` + set LABEL = `pstamptool -pendingreq -dbname pstamp | grep label | tail -n 1 | awk '{print $3}'` + + if ($VERBOSE != "") pstamptool -pendingreq -dbname pstamp -req_id $REQ_ID + + # run the parser on the previously generated entry + echo pstamp_parser_run.pl --dbname pstamp $VERBOSE --req_id $REQ_ID --uri $URI --product $PRODUCT --outdir $OUTDIR --label $LABEL + pstamp_parser_run.pl --dbname pstamp $VERBOSE --req_id $REQ_ID --uri $URI --product $PRODUCT --outdir $OUTDIR --label $LABEL +endif + +if ($STAGE2) then + if ($VERBOSE != "") pstamptool -pendingjob -dbname pstamp -req_id $REQ_ID + + set JOB_LIST = `pstamptool -pendingjob -dbname pstamp -req_id $REQ_ID | grep job_id | awk '{print $3}'` + + foreach JOB_ID ($JOB_LIST) + set J_TYPE = `pstamptool -pendingjob -dbname pstamp -req_id $REQ_ID -job_id $JOB_ID | grep jobType | awk '{print $3}'` + set ROWNUM = `pstamptool -pendingjob -dbname pstamp -req_id $REQ_ID -job_id $JOB_ID | grep rownum | awk '{print $3}'` + set OUTBASE = `pstamptool -pendingjob -dbname pstamp -req_id $REQ_ID -job_id $JOB_ID | grep outputBase | awk '{print $3}'` + set OPTIONS = `pstamptool -pendingjob -dbname pstamp -req_id $REQ_ID -job_id $JOB_ID | grep options | awk '{print $3}'` + + echo pstamp_job_run.pl --dbname pstamp $VERBOSE --job_id $JOB_ID --job_type $J_TYPE --rownum $ROWNUM --output_base $OUTBASE --options $OPTIONS + pstamp_job_run.pl --dbname pstamp $VERBOSE --job_id $JOB_ID --job_type $J_TYPE --rownum $ROWNUM --output_base $OUTBASE --options $OPTIONS + end +endif + +if ($STAGE3) then + if ($VERBOSE != "") pstamptool -completedreq -dbname pstamp + + # this should match the one we created above + set REQ_ID_X = `pstamptool -completedreq -dbname pstamp | grep req_id | tail -n 1 | awk '{print $3}'` + + if ($REQ_ID_X != $REQ_ID) then + echo "ERROR: I got back the wrong req_id: $REQ_ID_X vs $REQ_ID" + exit 2 + endif + + set REQ_TYPE = `pstamptool -completedreq -dbname pstamp | grep reqType | tail -n 1 | awk '{print $3}'` + set REQ_FILE = `pstamptool -completedreq -dbname pstamp | grep uri | tail -n 1 | awk '{print $3}'` + set REQ_NAME = `pstamptool -completedreq -dbname pstamp | grep name | grep -v username | tail -n 1 | awk '{print $3}'` + set PRODUCT = `pstamptool -completedreq -dbname pstamp | grep outProduct | tail -n 1 | awk '{print $3}'` + set OUTDIR = `pstamptool -completedreq -dbname pstamp | grep outdir | tail -n 1 | awk '{print $3}'` + + echo request_finish.pl --dbname pstamp $VERBOSE --req_id $REQ_ID --req_type $REQ_TYPE --req_file $REQ_FILE --req_name $REQ_NAME --product $PRODUCT --outdir $OUTDIR + request_finish.pl --dbname pstamp $VERBOSE --req_id $REQ_ID --req_type $REQ_TYPE --req_file $REQ_FILE --req_name $REQ_NAME --product $PRODUCT --outdir $OUTDIR +endif + +## show the status of finished jobs: +psstatus --finished Index: /branches/eam_branches/ipp-pstamp-20260421/pstamp/test/pstamptest.reset.sh =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/pstamp/test/pstamptest.reset.sh (revision 43092) +++ /branches/eam_branches/ipp-pstamp-20260421/pstamp/test/pstamptest.reset.sh (revision 43092) @@ -0,0 +1,50 @@ +#!/bin/csh + +## reset the pstamp database on localhost +## DO NOT run this on any machine with the name like ipp* +hostname | grep ipp +if ($status == 0) then + echo "DO NOT run on any machine with a name like 'ipp*'" + exit 2 +endif + +## NOTE: remove output files if re-running +## NOTE: remove datastore products, e.g., dsreg --rm --del web_1 --product pstampresults + +## assumes we have +# dsprodtool --add pstampresults --type PSRESULTS --description "test pstamp product" --ps0 req_id + +# remove the filesets listed under pstampresults: +set dsfiles = `dsproductls --uri http://dstest/ds/pstampresults/ | grep -v "^#" | awk '{print $2}'` +foreach f ($dsfiles) + echo dsreg --rm --del $f --product pstampresults + dsreg --rm --del $f --product pstampresults +end +echo "deleted old filesets from pstampresults" + +set workdir = `ppConfigDump -dump-site - | grep PSTAMP_WORKDIR | awk '{print $3}'` +set year = `date +%Y` +set month = `date +%m` +set day = `date +%d` + +foreach f ($workdir/$year/$month/$day/*) + echo rm -rf $f + rm -r f$f +end + +mysql -h localhost -u ipp -p pstamp < ~/src/panstarrs/ipp-pstamp-20260424/pstamp/doc/droptables.sql +mysql -h localhost -u ipp -p pstamp < ~/src/panstarrs/ipp-pstamp-20260424/pstamp/doc/tabledefs.sql + +pstamptool -dbname pstamp -adduser -set_user magnier -set_domain hawaii.edu -set_accessLevel 1 -set_defaultProduct pstampresults -set_defaultLabel EAM +pstamptool -dbname pstamp -adddomain -set_domain ifa.hawaii.edu -set_accessLevel 2 -set_defaultProduct ipp-pstamp-results -set_defaultLabel ifa-stamps +pstamptool -dbname pstamp -addproject -name simtest -imagedb simtest -inst SIMTEST -telescope SIMTEST + +# accessLevel must be set for the relevant project +pstamptool -dbname pstamp -addaccesslevel -set_project_name simtest -set_accessLevel 1 + +# alternate way to set accessLevel +# pstamptool -dbname pstamp -addaccesslevel -set_proj_id 1 -set_accessLevel 3 + +# default product: +# dsprodtool --add pstampresults --type PSRESULTS --description "test pstamp product" --ps0 req_id + Index: /branches/eam_branches/ipp-pstamp-20260421/pstamp/test/pstamptest.run.sh =================================================================== --- /branches/eam_branches/ipp-pstamp-20260421/pstamp/test/pstamptest.run.sh (revision 43092) +++ /branches/eam_branches/ipp-pstamp-20260421/pstamp/test/pstamptest.run.sh (revision 43092) @@ -0,0 +1,67 @@ +#!/bin/csh + +## this script runs a test sequence of postage stamp requests and processes them. + +## This script requires the pstamp database to exist, but the pstamp +## pantasks should not be running, or the labels used here should not +## be added to the pstamp pantasks + +## These commands define access control for a specific user and/or domain +if (0) then + ./reset.sh +endif + +# generate the faked web requests: + +if (0) then + # by exp_name (raw & chip) + pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --pixcenter --x 100 --y 100 --width 51 --height 51 --req_type byexp --stage raw --id sim.003.001 --component XY00 --survey bypass --jpeg --allroi + pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --pixcenter --x 100 --y 100 --width 10 --height 10 --req_type byexp --stage chip --id sim.003.000 --component XY00 --survey bypass --jpeg --allroi + pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --pixcenter --x 426 --y 1017 --width 10 --height 10 --req_type byexp --stage warp --id sim.003.005 --component skycell.1 --survey bypass --jpeg --allroi + + # by database ID (chip, warp, stack) + pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --pixcenter --x 100 --y 100 --width 10 --height 10 --req_type byid --stage chip --id 20 --component XY00 --survey bypass --jpeg --allroi + pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --pixcenter --x 426 --y 1017 --width 51 --height 51 --req_type byid --stage warp --id 18 --component skycell.1 --survey bypass --jpeg --allroi + pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --pixcenter --x 100 --y 100 --width 10 --height 10 --req_type byid --stage stack --id 13 --survey bypass --jpeg --allroi + pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --pixcenter --x 426 --y 1017 --width 51 --height 51 --req_type byid --stage diff --id 11 --component skycell.8 --survey bypass --jpeg --allroi +endif + +# select by exposure name, location by coordinates: +# OK pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.016608 --dec 30.026109 --width 50 --height 50 --req_type byexp --stage warp --id sim.003.000 --component skycell.4 --survey bypass --jpeg --allroi +# OK pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.016608 --dec 30.026109 --width 50 --height 50 --req_type byexp --stage chip --id sim.003.000 --component XY11 --survey bypass --jpeg --allroi + +# select by ID, location by coordinates: +# OK pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.016608 --dec 30.026109 --width 50 --height 50 --req_type byid --stage chip --id 20 --survey bypass --jpeg --allroi +# OK pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.016608 --dec 30.026109 --width 50 --height 50 --req_type byid --stage warp --id 18 --survey bypass --jpeg --allroi +# OK pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.016608 --dec 30.026109 --width 50 --height 50 --req_type byid --stage stack --id 13 --survey bypass --jpeg --allroi +# OK pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.085154 --dec 30.106472 --width 51 --height 51 --req_type byid --stage diff --id 11 --component skycell.8 --survey bypass --jpeg --allroi +# OK pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.085154 --dec 30.106472 --width 51 --height 51 --req_type byid --stage diff --id 11 --survey bypass --jpeg --allroi + +# select by coords: +# OK pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.016608 --dec 30.026109 --width 50 --height 50 --req_type bycoord --stage chip --survey bypass --jpeg --allroi +# OK pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.016608 --dec 30.026109 --width 50 --height 50 --req_type bycoord --stage raw --survey bypass --jpeg --allroi + +# this fails if TESS is not defined in the ipprc system (site.config) +# it also generates 5 of the 6 possible warps (for the basic simmosaic test) because one has no valid pixels at the requested location +# OK pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.016608 --dec 30.026109 --width 50 --height 50 --req_type bycoord --stage warp --survey bypass --jpeg --allroi --tess_id TESS +# OK pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.071935 --dec 30.119980 --width 50 --height 50 --req_type bycoord --stage stack --survey bypass --jpeg --allroi --tess_id TESS +# OK pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.085154 --dec 30.106472 --width 50 --height 50 --req_type bycoord --stage diff --survey bypass --jpeg --allroi --tess_id TESS + +set reqList = (`pstamptool -pendingreq -dbname pstamp | grep req_id | awk '{print $3}'`) +set i = 1 +while ($i <= $#reqList) + ./proc_next_stamp.sh + @ i++ +end + +# failures: + +# the location below on the specified warp has no valid pixels so the stamp creation exits with an error (fault 30 ?) +# XX pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --pixcenter --x 100 --y 100 --width 10 --height 10 --req_type byid --stage warp --id 18 --component skycell.1 --survey bypass --jpeg --allroi + +# the TESS_ID must be specified for bycoord requests from (warp,stack,diff) +# XX pstamp_webrequest.pl --label TEST --dbname pstamp --dbserver localhost --username magnier@hawaii.edu --email magnier@hawaii.edu --project simtest --ra 270.016608 --dec 30.026109 --width 50 --height 50 --req_type bycoord --stage warp --survey bypass --jpeg --allroi + +# send a warp run to be cleaned: +# warptool -dbname simtest -updaterun -warp_id 17 -set_state goto_cleaned +# warptool -dbname simtest -updaterun -warp_id 18 -set_label goto_cleaned -set_state goto_cleaned