Index: /trunk/gpc1_test_suite/fetch_detrend.pl =================================================================== --- /trunk/gpc1_test_suite/fetch_detrend.pl (revision 24368) +++ /trunk/gpc1_test_suite/fetch_detrend.pl (revision 24368) @@ -0,0 +1,121 @@ +#!/usr/bin/perl -w + +############################################################################## +# Script to track down the .fits images for the detrend images needed to +# reduce data. This does not magically tell you what detrends you need. I'm +# sure that information is in a database somewhere, but I don't know how to +# get to it. Instead, you specify keywords to match the detrend images, and +# it finds that. My method for identifying these keywords comes from ppImage +# on the raw .fits images for an exposure using: +# +# ppImage -file -dbname gpc1 -recipe PPIMAGE CHIP +# +# and then searching the output chip file's header to see what the +# DETREND.MASK, DETREND.DARK, and DETREND.FLAT keywords request. These need +# to be truncated to give the keyword to search for (for example, +# DETREND.MASK = 'GPC1.MASK.20090219.XY10.fits' => GPC1.MASK.20090219 +############################################################################## + +############################################################################## +# The options specify the keywords for each of the detrend steps we care about +############################################################################## +use Getopt::Std; +getopt('d:m:f:h',\%opt); + +if (exists($opt{h})) { + print STDERR "Usage: fetch_detrend.pl [-h] [-f ] [-d ] [-m ]\n"; + exit(1); +} + +############################################################################## +# Set up the database handle, and connect to "gpc1". ######################### +############################################################################## +use DBI; +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; +$dbname = 'gpc1'; +$dbserver = 'ipp004'; +$dbuser = 'ipp'; +$dbpass = 'ipp'; +$db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + + +############################################################################## +# We have three detrend stages that we need images for, but they can all be +# treated in nearly identical ways. For ease of code maintainance, I've put +# all of them into a single loop. +# +# To make use of that loop, define the stages, and the associated flags. +# +# I apologize for using plural words as arrays, and their singular as an +# element. Some people find it confusing, but it's more intuitive to me. +############################################################################## + +@detrend_stages = ('DARK','MASK','FLAT'); +@flags = ('d','m','f'); + +for ($i = 0; $i <= $#detrend_stages; $i++) { + $detrend_stage = $detrend_stages[$i]; + $flag = $flags[$i]; + + # Check to see that we have a keyword to search for this stage + if (exists($opt{${flag}})) { + # Make the output directory if it doesn't exist. + unless (-d "./${detrend_stage}") { + system("mkdir ./${detrend_stage}"); + } + + print STDERR "$flag -?-> $opt{$flag} -> ${detrend_stage}\n"; + + # The "final" detrends can either live in the detRegisteredImfile + # table or in the detNormalizedImfile table. We search one, and if + # we found no results, we try the other one. I'd like a more elegant + # solution, but this works. + $sth = "SELECT uri FROM detRegisteredImfile WHERE " . + "uri LIKE '%$opt{$flag}%'"; + $ref = $db->selectall_arrayref( $sth ) || + die "Unable to execute SQL: $DBI::errstr\n"; + + if ($#{ $ref } == -1) { + # This is where we're searching the other one. + $sth = "SELECT uri FROM detNormalizedImfile WHERE " . + "uri LIKE '%$opt{$flag}%'"; + $ref = $db->selectall_arrayref( $sth ) || + die "Unable to execute SQL: $DBI::errstr\n"; + + # If we still didn't find anything, let's just panic and die, + # because there's clearly some problem somewhere. + if ($#{ $ref } == -1) { + die "Could not find that detrend: $flag : $opt{$flag}\n"; + } + } + + # Let the user know we've done something, by counting our matches. + print STDERR + "$detrend_stage found $#{ $ref } matches for %$opt{$flag}\n"; + + # Loop over each match, and copy the real image file to a directory + # matching the stage of detrending it will be used for. This is + # basically the same as what fetch_rawExp.pl does. There should be + # ~60 of these as well, as you should get one for each detector chip. + foreach $row_ref (@{ $ref }) { + $uri = shift( @{ $row_ref }); + chomp($real_file = + `neb-locate --server http://ipp004/nebulous/ --path $uri`); + print STDERR "$uri -> ./${detrend_stage}/${real_file}\n"; + system("cp $real_file ./${detrend_stage}/"); + } + } # Finish searching for this stage. + else { + # Not all stages need to be done at once, or even at all (since the + # flats will probably change with filter, but the mask and dark do not + # necessarily need to change), but it's always good to tell the user + # that you're at least paying attention to them. + print STDERR "No keyword for $detrend_stage, skipping.\n"; + } +} # Continue to next detrend stage, and then close down. + +$db->disconnect(); Index: /trunk/gpc1_test_suite/fetch_rawExp.pl =================================================================== --- /trunk/gpc1_test_suite/fetch_rawExp.pl (revision 24368) +++ /trunk/gpc1_test_suite/fetch_rawExp.pl (revision 24368) @@ -0,0 +1,123 @@ +#!/usr/bin/perl -w + +############################################################################## +# Script to find the raw .fits images for a given exposure name (like those of +# the form "gTMJDgEXPNo" with TMJD being the truncated modified Julian date, +# and EXPN being the exposure number). Written to solve two problems. First, +# I had no idea how the database was set up, and this was a good way to learn +# that. Second, I needed a way to copy some images to a sandbox to play with +# them safely. +############################################################################## + +############################################################################## +# Set up the database handle, and connect to "gpc1". ######################### +############################################################################## +use DBI; +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; +$dbname = 'gpc1'; +$dbserver = 'ipp004'; +$dbuser = 'ipp'; +$dbpass = 'ipp'; +$db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + + +############################################################################## +# Loop over the input exposure names, and find them in the database.########## +############################################################################## + +# Save the information we find out about the images to a list file, because +# maybe we'll care about that in the future. You can never tell. +open(LIST,">>./rawExp.list") || die "Could not write my list file.\n"; + +while (<>) { + chomp; + $x_name = $_; + + # Grab the same information as the ippMonitor webpage + $sth = "SELECT exp_id,exp_name,dateobs,ra,decl,filter,exp_time,airmass, " . + "bg,bg_stdev,bg_mean_stdev,comment from rawExp WHERE " . + "exp_name like '%$x_name%' "; + $data_ref = $db->selectall_arrayref( $sth ) || + die "Unable to execute SQL: $DBI::errstr\n"; + + # Store that ippMonitor information in the list file. This adds a quick + # header so that the list is self documenting. + print LIST "#$x_name\n"; + print LIST "#exp_id\texp_name\tdate/time\tRA\tDEC\tfilter\texp_time\t" . + "airmass\tbkg\tsigbkg\tbg_mean_stdev\tcomment\n"; + print STDERR "Working on $x_name\n"; + + # We should only receive one row hit from a fully specified exp_name + # (gTMJDgEXPNo), but the query should match just as well if you specify + # only the date (gTMJDg), so you may possibly receive multiple rows. For + # such an example, you'll receive N rows for the N exposures taken on + # that day. + foreach $row_ref (@{ $data_ref }) { + $exp_id = ${ $row_ref}[0]; + + # Write out the ippMonitor data to the list file. + $list_out = join "\t", @{ $row_ref }; + print LIST "$list_out\n"; + + # Make a directory to hold the .fits images. These are named by the + # exposure id (rawExp.exp_id) to ensure they don't overlap. The list + # file allows these ids to be referenced to the exposure name. + system("mkdir $exp_id"); + print STDERR "\tID: $exp_id\n"; + + # Each detector has information in the database as well. This is + # stored in yet another list, to ensure no loss of information + # compared to ippMonitor (this is the stuff that shows up when you + # click on an exp_id). This is also where we find out the uri + + $c_sth = "SELECT exp_name,class_id,bg,bg_stdev,bg_mean_stdev, " . + "uri FROM rawImfile WHERE " . + "exp_id LIKE '$exp_id'"; # I'm not sure why I used LIKE here. + $chip_ref = $db->selectall_arrayref( $c_sth ) || + die "Unable to execute SQL: $DBI::errstr\n"; + + # Write header for rawImfile.list. + open(CHIPLIST,">>./${exp_id}/rawImfile.list") || + die "Could not write my chip list file: $exp_id.\n"; + print CHIPLIST "#$exp_id\n"; + print CHIPLIST "#exp_name.class\tbkg\tsigbkg\tbg_mean_stdev\turi\n"; + + # This should yield something ~60 rows, which are the individual + # detector chips. Each one is identified, a verbose statement sent + # to the terminal, the rawImfile.list updated, and then the uri + # converted to a real file. + foreach $chip_row_ref (@{ $chip_ref}) { + $c_exp_name = ${ $chip_row_ref }[0]; + $c_class_id = ${ $chip_row_ref }[1]; + $c_uri = ${ $chip_row_ref }[-1]; + print STDERR "\t\t${c_exp_name}.${c_class_id} -> $c_uri\n"; + + $chiplist_out = join "\t", @{ $chip_row_ref }; + $chiplist_out =~ s/\t/./; + print CHIPLIST "$chiplist_out\n"; + + # Use neb-locate to find the real image file, and copy it to the + # data directory set up above. + chomp($real_file = + `neb-locate --server http://ipp004/nebulous/ --path $c_uri`); + system("cp $real_file ./${exp_id}/"); + + } + close(CHIPLIST); + # This closes out this exp_id... + } + # and this closes out the exp_name matches. +} +close(LIST); +# and now we're all done with everything we set out to do. If there haven't +# been any errors, there should be (N > 0) directories named after the exp_id +# they contain the data for. There should be a master output list with the +# database parameters for the exposures, as well as exposure output lists +# with the database parameters for the raw images. Most importantly, though +# the individual .fits images are now copied out of the nebulous system. + +$db->disconnect(); Index: /trunk/gpc1_test_suite/gpc1_test.auto =================================================================== --- /trunk/gpc1_test_suite/gpc1_test.auto (revision 24368) +++ /trunk/gpc1_test_suite/gpc1_test.auto (revision 24368) @@ -0,0 +1,16 @@ +automate MULTI + +automate METADATA + name STR CHIP + regular STR "chiptool -updaterun -set_label @DBNAME@ -dbname @DBNAME@ -inst GPC1" +END + +automate METADATA + name STR STACK + regular STR "stacktool -definebyquery -all -label @DBNAME@ -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@" +END + +#automate METADATA +# name STR DIFF +# regular STR "difftool -definewarpstack -label @DBNAME@ -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@" +#END Index: /trunk/gpc1_test_suite/gpc1_test.pro =================================================================== --- /trunk/gpc1_test_suite/gpc1_test.pro (revision 24368) +++ /trunk/gpc1_test_suite/gpc1_test.pro (revision 24368) @@ -0,0 +1,55 @@ +# gpc1_test.pro : automated test to combine the 2009/06/02 MD07 images +# since converted into a general way to push real GPC1 data +# through the IPP using pantasks. +# Based in no small part from the simtest.pro module + + +macro gpc1_test + if ($0 != 5) + echo "USAGE: gpc1_test " + echo " init = run : only run the analysis tasks" + echo " inject : recreate the database and reinject the images" + echo " new : restart from scratch" + break + end + + $dbname = $1 + $hostname = $2 + $datadir = $3 + $init = $4 + + if (("$init" != "run") && ("$init" != "inject") && ("$init" != "new")) + echo "USAGE: gpc1_test " + echo " init = run : only run the analysis tasks" + echo " inject : recreate the database and reinject the images" + echo " new : restart from scratch" + break + end + + if (("$init" == "new") || ("$init" == "inject")) + exec pxadmin -dbname $dbname -delete + exec pxadmin -dbname $dbname -create + +# exec skycells -mode LOCAL -scale 0.2 -nx 12 -ny 12 -size 4 4 -fix-ns -center 213.146 53.417 -D CATDIR /data/ipp022.0/md07_test/TESS + exec /home/panstarrs/watersc1/bin/mk_project_database.pl -d $dbname -w $datadir + end + + module pantasks.pro + module automate.pro + + module.tasks + + controller host add $hostname + + add.database $dbname + + automate.load gpc1_test.auto GPC1 $dbname + + detrend.off + + add.label $dbname + + + run +end + Index: /trunk/gpc1_test_suite/mk_project_database.pl =================================================================== --- /trunk/gpc1_test_suite/mk_project_database.pl (revision 24368) +++ /trunk/gpc1_test_suite/mk_project_database.pl (revision 24368) @@ -0,0 +1,252 @@ +#!/usr/bin/perl -w +# +# Script to inject data and detrend images into a database. Originally written +# to make the creation of test datasets from raw exposures "easy." + + +############################################################################### +# Load configuration settings. The defaults are set up for my MD07 test suite, +# but can be over-ridden without much difficulty. +############################################################################### +use Getopt::Std; + +$opt{i} = 'GPC1'; +$opt{t} = 'PS1'; +$opt{d} = 'md07_20090602'; +$opt{w} = '/data/ipp022.0/md07_test'; +$opt{f} = 'i'; +getopts('i:t:d:w:f:',\%opt); + +$inst = $opt{i}; +$telescope = $opt{t}; +$dbname = $opt{d}; +$workdir = $opt{w}; +$filter = $opt{f}; + +print STDERR "SETTINGS:\n"; +print STDERR "\tINSTRUMENT: $inst\n"; +print STDERR "\tTELESCOPE: $telescope\n"; +print STDERR "\tDATABASE: $dbname\n"; +print STDERR "\tWORKING DIR: $workdir\n"; +print STDERR "\tFILTER: $filter\n"; +print STDERR "\n"; + +############################################################################### +# Set up the directories we'll be scanning for data. The detrends are expected +# to all be in individual directories. This doesn't gracefully handle multiple +# filters (which change the FLAT), so for right now, each filter needs to be +# loaded individually. The data directories are named after the exposure ID, +# which makes them easy-to-find digits. +# +# Note that you can easily put multiple exposures into a single directory (for +# both the data and the detrend). We scan later to ensure everything gets +# included. +############################################################################### +print STDERR "DIRECTORIES:\n"; +@detrend_dirs_tmp = ("$workdir/ASTROM","$workdir/DARK", + "$workdir/FLAT","$workdir/MASK"); +@detrend_dirs = (); +foreach $ddir (@detrend_dirs_tmp) { + if (-d $ddir) { + push @detrend_dirs, $ddir; + print STDERR "\tFound detrend directory: $ddir\n"; + } +} +print STDERR "\n"; + +@data_dirs = (); +@files = <$workdir/*>; +foreach $f (@files) { + if (-d $f) { + if ($f =~ /^$workdir\/\d+$/) { + push @data_dirs, $f; + print STDERR "\tFound data directory: $f\n"; + } + } +} +print STDERR "\n"; + +############################################################################### +# Process the detrend images. +############################################################################### +print STDERR "DETRENDS:\n"; +foreach $ddir (@detrend_dirs) { + print STDERR "\tScanning $ddir...\n"; + + %detrend_data = (); + + @detfiles = <$ddir/*.fits>; + push @detfiles, <$ddir/*.asm>; + + # Get the exposure name root and the class id for each file in the + # directory. + $det_type = $ddir; + $det_type =~ s%^.*/(.*)$%$1%; + foreach $f (@detfiles) { + ($exp_name,$class_id) = find_expname_classid($f); + $detrend_data{$exp_name}{$class_id} = "$f"; + } + + foreach $exp_name (keys %detrend_data) { + print STDERR "\t\tAdding $exp_name...\n"; + # Build the necessary dettool command string: + $det_id = ''; + if ($det_type eq 'ASTROM') { + $filelevel = "fpa"; + } + else { + $filelevel = "CHIP"; + } + $cmd = "dettool -dbname $dbname " . + "-register_detrend " . + "-det_type $det_type -filelevel $filelevel " . + "-inst $inst -telescope $telescope"; + if ($det_type eq 'FLAT') { + $cmd .= " -filter $filter"; + } + if ($det_type eq 'ASTROM') { + $cmd .= " -use_begin 2008/01/01"; + } + + # Run dettool registration step, and catch the det_id: + print STDERR "\t\t$cmd\n"; + open(DETTOOL,"$cmd |") || die "Failed to run $cmd\n"; + while () { + chomp; + print "DETTOOL:$_\n"; + $_ =~ s/^\s+//; + if ($_ =~ /^det_id/) { + $det_id = (split /\s+/, $_)[2]; + } + } + close(DETTOOL); + if ($det_id eq '') { # Or not, and shut down everything. + die "Did not recieve a det_id.\n"; + } + + # Load the individual image files: + foreach $class_id (sort (keys %{ $detrend_data{$exp_name} })) { + $cmd = "dettool -dbname $dbname " . + "-register_detrend_imfile -det_id $det_id " . + "-class_id $class_id -uri $detrend_data{$exp_name}{$class_id}"; + + print STDERR "\t\t\t$cmd\n"; + system($cmd); + } + + # Tell dettool that we're done with this exposure. + $cmd = "dettool -dbname $dbname " . + "-updatedetrun -state stop -det_id $det_id"; + + print STDERR "\t\t$cmd\n"; + system($cmd); + + } +} +print STDERR "\n"; + +############################################################################## +# Process the data images, doing essentially the same stuff. +############################################################################## +print STDERR "DATA:\n"; +foreach $ddir (@data_dirs) { + print STDERR "\tScanning $ddir...\n"; + + %image_data = (); + + @imfiles = <$ddir/*.fits>; + + # Get the exposure name root and teh class id for each file in the + # directory. + foreach $f (@imfiles) { + ($exp_name,$class_id) = find_expname_classid($f); + $image_data{$exp_name}{$class_id} = "$f"; + } + + foreach $exp_name (keys %image_data) { + print STDERR "\t\tAdding $exp_name...\n"; + # Build the necessary pxinject command string: + $exp_id = ''; + $cmd = "pxinject -dbname $dbname " . + "-newExp -tmp_exp_name $exp_name -workdir $workdir " . + "-tmp_inst $inst -tmp_telescope $telescope -tess_id $workdir/TESS"; + + # Run pxinject registration step, and catch the exp_id: + print STDERR "\t\t$cmd\n"; + open(PXINJECT,"$cmd |") || die "Failed to run $cmd\n"; + while () { + chomp; + $_ =~ s/^\s+//; + if ($_ =~ /^exp_id/) { + $exp_id = (split /\s+/, $_)[2]; + } + } + close(PXINJECT); + if ($exp_id eq '') { # Or not, and shut down everything. + die "Did not receive an exp_id.\n"; + } + + # Load the individual image files: + foreach $class_id (sort (keys %{ $image_data{$exp_name} } )) { + $cmd = "pxinject -dbname $dbname " . + "-newImfile -exp_id $exp_id " . + "-tmp_class_id $class_id " . + "-uri $image_data{$exp_name}{$class_id}"; + + print STDERR "\t\t\t$cmd\n"; + system($cmd); + } + + # Tell pxinject that we're done with this exposure. + $cmd = "pxinject -dbname $dbname " . + "-updatenewExp -state run -exp_id $exp_id"; + + print STDERR "\t\t$cmd\n"; + system($cmd); + } +} +print STDERR "\n"; + + +############################################################################## +# End of script! +############################################################################## +print STDERR "DATABASE INJECTED SUCCESSFULLY\n"; + + +############################################################################## +# Subroutine to read a full filename, and extract out the exposure name and +# class id. Basically just a quick regexp replace. +############################################################################## +sub find_expname_classid { + my $f = shift; + + $f =~ s%^.*/(.*)%$1%; + + if ($f =~ /\.asm$/) { + $exp_name = $f; + $class_id = 'fpa'; + } + elsif ($f =~ /\.XY\d\d.fits/) { + $f =~ s/(.*?)\.XY(\d\d).fits/$1 XY$2/; + ($exp_name,$class_id) = split /\s/, $f; + } + elsif ($f =~ /\.ota\d\d.fits/) { + $f =~ s/(.*?)\.ota(\d\d).fits/$1 ota$2/; + ($exp_name,$class_id) = split /\s/, $f; + } + else { + die "Cannot handle file: $f\n"; + } + return($exp_name,$class_id); +} + + + + + + + + + +