Index: /trunk/tools/iasc/README =================================================================== --- /trunk/tools/iasc/README (revision 37326) +++ /trunk/tools/iasc/README (revision 37326) @@ -0,0 +1,70 @@ +# IASC is the asteroids/comets outreach program +# Ask Richard Wainscoat about it +# SVN: https://svn.pan-starrs.ifa.hawaii.edu/repo/ipp/trunk/tools/iasc + +### Usage +1) Select the set of exposures to be published + e.g. o6897g0312o o6897g0328o o6897g0344o o6897g0360o +2) Choose a name for the target directory (usually the date when the observations + were made) + e.g. 20140828 +3) Execute: + ./RUN.sh + e.g. + ./RUN.sh 20140828 o6897g0312o o6897g0328o o6897g0344o o6897g0360o +4) Send an e-mail to the people who requested the set of exposures, telling them + where to find the set + + +### Details + +To get chip processed image files with updated astrometry we have the postage stamp +server make full image stamps for each chip. Then we download the files and change +the names to remove the stuff that the pstamp server needs to add. + +The process is + +1. Make the destination directory. I have been using the date. For example + mkdir 20120412 + +For each exposure + postreq + # wait for request to finish (check pstamp status page) + getit + +2. rsync the directory to /data/ipp022.0/ps1-outreach: + rsync -avz YYYYMMDD ipp@ipp0222:/data/ipp022.0/ps1-outreach/ + To speed it up: + rsync -av YYYYMMDD ipp@ipp0222:/data/ipp022.0/ps1-outreach --exclude 'YYYYMMDD/*/*.tmp' + +3. The directory is visible to epo folks as + rsync://ipp0222.ifa.hawaii.edu/ps1-outreach/YYYYMMDD + Test it with: + rsync -avz rsync://ps1epo@ipp0222.ifa.hawaii.edu/ps1-outreach/20130308 . + Password can be found in ipp022:/etc/rsyncd.secrets + +4. Make sure that the smfs have been published to the datastore + ./check_if_smfs_have_been_published.py o6359g0309o o6359g0330o o6359g0351o o6359g0372o + +5. Tell the persons who made the request that all is good + +###### +For example + (ippc18:~/outreach) ./postreq o6029g0262o + running psmkreq --req_type byexp --stage chip --pixcenter --x 0 --y 0 --width 0 --height 0 --id o6029g0262o --req_name o6029g0262o.20120412T201043 + running pstamptool -dbname ippRequestServer -dbserver ippc17 -label WEB -addreq -uri /home/panstarrs/bills/outreach/o6029g0262o.20120412T201043.fits + posted o6029g0262o.20120412T201043 req_id: 160657 + results will be at http://datastore.ipp.ifa.hawaii.edu/pstampresults/o6029g0262o.20120412T201043 + + ./getit o6029g0262o.20120412T201043 20120412 + # lots of noise from dsget + + # files are in 20120412/o6029g0262o + +./check_if_smfs_have_been_published.py o6359g0309o o6359g0330o o6359g0351o o6359g0372o +o6359g0309o: OK (2 published) +o6359g0330o: OK (2 published) +o6359g0351o: OK (2 published) +o6359g0372o: OK (2 published) + +NOTE : the directory in which this is run needs to be globally visible via NFS. Index: /trunk/tools/iasc/RUN.sh =================================================================== --- /trunk/tools/iasc/RUN.sh (revision 37326) +++ /trunk/tools/iasc/RUN.sh (revision 37326) @@ -0,0 +1,46 @@ +#!/bin/bash + +targetdir=$1 +shift +exposures=$* + +echo "Creating targetdir [$targetdir]" +mkdir -p $targetdir +mkdir -p $targetdir/smf + +images="" +for exposure in $exposures; do + echo "Requesting images for $exposure" + image=`./postreq $exposure | grep datastore.ipp.ifa.hawaii.edu | sed 's/^.*\///'` + images="$images $image" + echo "Requesting smf for $exposure" + ./get_smf.sh $exposure $targetdir +done + +todelete="" +for image in $images; do + status=1 + while [ $status -ne 0 ]; do + sleep 30 + echo "Copying images for $image" + ./getit $image $targetdir + status=$? + done + todelete="$todelete $image.fits" +done +echo "All images copied locally" + +echo "Rsyncing: rsync -av $targetdir ipp@ipp0222:/data/ipp022.0/ps1-outreach/" +rsync -av $targetdir ipp@ipp0222:/data/ipp022.0/ps1-outreach/ +rsync -av $targetdir ipp@ipp0222:/data/ipp022.0/ps1-outreach/ + +echo +echo "Done" +echo "Dataset can be copied from rsync://ps1epo@ipp0222.ifa.hawaii.edu/ps1-outreach/$targetdir" +echo "Command line:" +echo " rsync -avz rsync://ps1epo@ipp0222.ifa.hawaii.edu/ps1-outreach/$targetdir ." +echo +echo "Command to run to cleanup local directory:" +echo " rm -rf $targetdir $todelete" +echo + Index: /trunk/tools/iasc/get_smf.sh =================================================================== --- /trunk/tools/iasc/get_smf.sh (revision 37326) +++ /trunk/tools/iasc/get_smf.sh (revision 37326) @@ -0,0 +1,12 @@ +#!/bin/bash + +# $1: expsore name +# $2: destination directory (date) + +pathbase=`echo "SELECT path_base FROM camProcessedExp JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE exp_name = '$1' " | mysql -h scidbm -u ipp -pipp gpc1 | grep -v path_base` + +source=`neb-ls -p $pathbase.smf` + +mkdir -p $2/smf +cp $source $2/smf/$1.smf + Index: /trunk/tools/iasc/getit =================================================================== --- /trunk/tools/iasc/getit (revision 37326) +++ /trunk/tools/iasc/getit (revision 37326) @@ -0,0 +1,49 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +my $dsroot = 'http://datastore.ipp.ifa.hawaii.edu/pstampresults'; + +my $fs = shift; +my $dest = shift; + +# set this if the files have already been downloaded +my $skip_download = 0; + +die "usage: $0 > 8; + die "dsgetfileset failed with $rc $status\n"; + } +} + +# read exp_name from results.mdc (keyword is FPA_ID) +my $grepout = `grep FPA_ID $destdir/results.mdc | tail -n 1`; +chomp $grepout; + +die "failed to find exp_name in results.mdc" unless $grepout; + +my (undef, undef, $exp_name) = split " ", $grepout; + +die "failed to find exp_name in results.mdc: $grepout" unless $exp_name; + +my $final_destdir = "$dest/$exp_name"; +rename $destdir, $final_destdir or die "failed to rename $destdir"; + +chdir $final_destdir or die "failed to chdir to $final_destdir"; + +system 'perl /data/ippc18.0/home/ipp/iasc.v2/rename.stamp.outputs.pl *'; + + Index: /trunk/tools/iasc/obsolete/check_if_smfs_have_been_published.py =================================================================== --- /trunk/tools/iasc/obsolete/check_if_smfs_have_been_published.py (revision 37326) +++ /trunk/tools/iasc/obsolete/check_if_smfs_have_been_published.py (revision 37326) @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +import MySQLdb +import sys + +def usage(program_name): + return """ +%s [exposure name] [...] + +Check if the smf of have been published to the datastore + +-h, -help, --help: + Displays this help and exits +""" + +def process_arguments(arguments): + if len(arguments)<2: + sys.stderr.write(usage(arguments[0])) + sys.exit(1) + index = 1 + if arguments[index] == "-h" or arguments[index] == "-help" or arguments[index] == "--help": + sys.stderr.write(usage(arguments[0])) + sys.exit(0) + return arguments[index:] + +if __name__ == "__main__": + exposures = process_arguments(sys.argv) + gpc1 = MySQLdb.connect('ippdb01', 'ipp', 'ipp', 'gpc1') + cursor = gpc1.cursor() + for exposure in exposures: + statement = "SELECT COUNT(distinct(distRun.dist_id)) FROM distRun JOIN distComponent USING(dist_id) JOIN camRun ON camRun.cam_id=distRun.stage_id JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE exp_name = '%s' AND distRun.stage='camera'" % exposure + cursor.execute(statement) + count=cursor.fetchall()[0][0] + if count != 0: + print "%s: OK (%d published)" % (exposure, count) + else: + print "%s: FAIL (%d published)" % (exposure, count) + Index: /trunk/tools/iasc/postreq =================================================================== --- /trunk/tools/iasc/postreq (revision 37326) +++ /trunk/tools/iasc/postreq (revision 37326) @@ -0,0 +1,36 @@ +#!/usr/bin/env perl + +# create a postage stamp request file to get full chip images for +# a given exposure +# post it to the server with label WEB + +use strict; +use warnings; +use POSIX qw( strftime ); + +my $exp_name = shift; +die "need exposure name\n" if !$exp_name; + +my $datestr = strftime "%Y%m%dT%H%M%S", gmtime; + +# request name must be unique add datetime +my $req_name = "$exp_name.$datestr"; + +# make the request file +my $cmd = "psmkreq --req_type byexp --stage chip --pixcenter --x 0 --y 0 --width 0 --height 0 --id $exp_name --req_name $req_name"; + +print "running $cmd\n"; +my $rc = system $cmd; + +die "psmkreq failed" if $rc; + +# add the request to the database +my $postcmd = "pstamptool -dbname ippRequestServer -dbserver ippc17 -label WEB -addreq -uri $ENV{PWD}/$req_name.fits"; + +print "running $postcmd\n"; + +my $req_id = `$postcmd`; +chomp $req_id; + +print "posted $req_name req_id: $req_id\n"; +print "results will be at http://datastore.ipp.ifa.hawaii.edu/pstampresults/$req_name\n"; Index: /trunk/tools/iasc/rename.stamp.outputs.pl =================================================================== --- /trunk/tools/iasc/rename.stamp.outputs.pl (revision 37326) +++ /trunk/tools/iasc/rename.stamp.outputs.pl (revision 37326) @@ -0,0 +1,12 @@ +#!/usr/bin/env perl + +my $verbose = 0; + +foreach my $name (@ARGV) { + next if !($name =~ /1_/); + my (undef, undef, undef, $new_name) = split "_", $name; + die "failed to extract new name from $name\n" unless $new_name; + print "$name $new_name\n" if $verbose; + rename $name, $new_name or die "failed to rename $name\n"; +} +