Index: trunk/tools/refstacktool =================================================================== --- trunk/tools/refstacktool (revision 30338) +++ trunk/tools/refstacktool (revision 30338) @@ -0,0 +1,351 @@ +#!/bin/env perl + +use strict; +use warnings; +use DBI; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use File::Basename; + + +my ($field, $filter, $label, $data_group, $dist_group, $tess_id, $workdir, $reduction, $end_stage); +my ($verbose); + +# parameters with default values +my $date_begin = '2009-04-01'; +my $date_end; +my $fwhm_min = 3; +my $fwhm_max; +my $n_inputs_min = 60; +my $n_inputs_max = 100; +my $tess_version = 'V2'; +my $input_data_group; + +my $dbname = "gpc1"; + +my $list = 0; +my $pretend = 0; +my $queue = 0; + +GetOptions( + 'list' => \$list, + 'queue' => \$queue, + 'pretend' => \$pretend, + 'field=s' => \$field, + 'filter=s' => \$filter, + 'fwhm_min=s' => \$fwhm_min, + 'fwhm_max=s' => \$fwhm_max, + 'input_data_group'=> \$input_data_group, + 'n_inputs_min=i'=> \$n_inputs_min, + 'n_inputs_max=i'=> \$n_inputs_max, + 'date_begin=s' => \$date_begin, + 'date_end=s' => \$date_end, + 'label=s' => \$label, + 'data_group=s' => \$data_group, + 'dist_group=s' => \$dist_group, + 'tess_version=s'=> \$tess_version, + 'tess_id=s' => \$tess_id, + 'workdir=s' => \$workdir, + 'reduction=s' => \$reduction, + 'end_stage=s' => \$end_stage, + 'verbose' => \$verbose, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "--field and --filter are required", -exitval => 2 ) if !$field or !$filter; + +pod2usage( -msg => "Only one of --queue and --pretend may be supplied ", -exitval => 2 ) if $queue and $pretend; + +my ($d, $m, $y) = (gmtime) [3,4,5]; +my $today = sprintf "%d%02d%02d", $y + 1900, $m + 1, $d; +print "today is $today\n" if $verbose; + +if (!$date_end) { + # default date_end is 30 days ago + my $then = time() - 30 * 86400; + my ($td, $tm, $ty) = (gmtime($then)) [3, 4, 5]; + $date_end = sprintf "%d-%02d-%02d", $ty + 1900, $tm + 1, $td; + print "date_end: $date_end\n" if $verbose; +} + + +my $ipprc = PS::IPP::Config->new(); +my $dbh = getDBHandle(); + +my $selection_args = " fwhm_major >= $fwhm_min"; + +$selection_args .= " AND fwhm_major <= $fwhm_max" if $fwhm_max; +$selection_args .= " AND dateobs >= '$date_begin'"; +$selection_args .= " AND date(dateobs) <= '$date_end'" if $date_end; + +$input_data_group = "'$field.%'"; + +my $query = "SELECT exp_id, exp_name, dateobs, fwhm_major, object, comment FROM camRun join camProcessedExp USING(cam_id) JOIN chipRun using(chip_id, data_group) join rawExp using(exp_id) where data_group like $input_data_group AND filter LIKE '$filter%' AND $selection_args ORDER BY fwhm_major, cam_id DESC LIMIT $n_inputs_max"; + +print STDERR "$query\n" if $verbose; + +my $stmt = $dbh->prepare($query); + +if (!$stmt->execute()) { + die "query failed\n"; +} + +my @exps; +my $n_exps = 0; +my $exp_id_last = 0; +while (my $exp = $stmt->fetchrow_hashref()) { + # avoid exposures processed multiple times + my $exp_id = $exp->{exp_id}; + next if $exp_id_last == $exp_id; + $exp_id_last = $exp_id; + + $n_exps++; + push @exps, $exp; +} + +if ($list) { + print " i exp_id exp_name dateobs fwhm_major comment\n"; + print "-------------------------------------------------------------------------------------------\n"; +} +my $max_fwhm = 0; +my $min_fwhm = 100000; +for (my $i = 0; $i < $n_exps; $i++) { + my $exp = $exps[$i]; + my $exp_id = $exp->{exp_id}; + my $exp_name = $exp->{exp_name}; + my $dateobs = $exp->{dateobs}; + my $fwhm_major = $exp->{fwhm_major}; + my $object = $exp->{object}; + my $comment = $exp->{comment}; + $max_fwhm = $fwhm_major if $fwhm_major > $max_fwhm; + $min_fwhm = $fwhm_major if $fwhm_major < $min_fwhm; + + printf "%3d %8d %12s %20s %10f %s\n", $i+1, $exp_id, $exp_name, $dateobs, $fwhm_major, $comment if $list; +} +print "Found $n_exps exposures for $field $filter.\n"; +print "Range of fhwm_major: $min_fwhm - $max_fwhm\n"; + +if ($n_exps < $n_inputs_min) { + print "Number of exposures is less than the minimum required: $n_inputs_min\n"; + exit 1; +} + +if ($queue or $pretend) { + if (!$label) { + $label = "$field.refstack.$today"; + } + $dist_group = $field if !$dist_group; + $tess_id = "$field.$tess_version" if !$tess_id; + $workdir = 'neb://@HOST@.0/gpc1/' . $label if !$workdir; + $end_stage = 'warp' if !$end_stage; + + my $command = "chiptool -dbname $dbname -definebyquery -set_label $label -set_workdir $workdir"; + $command .= " -set_dist_group $dist_group"; + $command .= " -set_data_group $data_group" if $data_group; + $command .= " -set_tess_id $tess_id"; + $command .= " -set_end_stage $end_stage"; + + $command .= " -pretend -simple" if $pretend; + for (my $i = 0; $i < $n_inputs_max and $i < $n_exps; $i++) { + my $exp = $exps[$i]; + my $c = "$command -exp_id $exp->{exp_id}"; + print "$c\n" if $verbose; + my $status = system $c; + if ($status) { + my $exit_status = $status >> 8; + die "chiptool returned $exit_status"; + } + } +} + +exit 0; + +sub getDBHandle { + my $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "DBSERVER"); + my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER"); + my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD"); + if (!$dbname) { + $dbname = metadataLookupStr($ipprc->{_siteConfig}, "DBNAME"); + } + + die "database configuration set up" unless defined($dbserver) and defined($dbuser) + and defined($dbpassword) and defined($dbname); + + my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; + + my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) + or die "Cannot connect to database.\n"; + + return $dbh; +} + +=pod + +=head1 NAME + +refstacktool - queue processing of selected exposures for input to a reference stack. + +=head1 SYNOPSIS + + refstacktool --field --filter [--list] [--queue | --pretend] [selection options] [run options] + +=head1 DESCRIPTION + +Executes a query of IPP database to select exposures from a given field to be queued for chip processing +based on various selection parameters. + +There are three modes of operation. By default it the number of matching +exposures and the range of fwhm_major values is listed. + +For example: + + % refstacktool --field MD05 --filter y + Found 99 exposures for MD05 y. + Range of fhwm_major: 3.133 - 4.97175 + +If the option --queue is supplied, chiptool is invoked to queue chipRuns for the selected exposues. +The image quality and date range for selected exposures may be controlled using the selection options +described below. + +The parameters for the chipRun may be controlled with the run options. + +=head1 OPTIONS + +=over 4 + +=item * --field + +The PS1 field to search for exposures. For example, MD05, STS, or M31. This value is used to construct +default values for various other parameters. In particular exposures are selected where data_group LIKE '$field%' +(XXX: should we use rawExp.comment?) + +Required. + +=item * --filter + +The filter. The supplied value is appended with a SQL wild card '%' and selection is performed with a "LIKE" +comparision. + +Required. + +=item * --list + +If supplied certain parameters of the selected exposures are displayed. + +Optional. + + +=item * --queue + +If the number of exposures is less than the value for n_inputs_min chiptool is run to +insert chipRuns for the selected exposures. + +Optional. + +=item * --pretend + +Like --queue but the arguments "--pretend --simple" are appended to the chiptool command. +(Only one of --queue or --pretend may be supplied.) + +Optional. + +=head1 SELECTION OPTIONS + +=item * --fwhm_max + +Select exposures where the value of fwhm_major for a previous camRun for the expousre is less than or +equal to the supplied value. fwhm_major is measured in pixels. + +Optional. + +=item * --fwhm_min + +Select exposures where the value of fwhm_major for a previous camRun for the expousre is greater than or +equal to the supplied value. + +Optional. The default value is 3 pixels. + +=item * --date_begin + +Select exposures taken after 00:00:00 UTC on the given date. The date format is YYYY-MM--DD style. + +Optional. The default value is '2009-04-01'. + +=item * --date_end + +Select exposures taken on or before the given date. + +Optional. The default value is 30 days prior to the current date. + +=item * --input_data_group + +Select exposures which were previously processed with chipRun and camRun that match the supplied data_group. + +Optional. The default value is "field.%. For example 'MD05%' + +=item * --n_inputs_min + +With --queue or --pretend only run the chiptool commands if the number of selected exposures is greater than +or equal to the supplied value. + +Optional. The default value is 60 exposures. + +=item * --n_inputs_max + +Limit the query to the supplied value. + +Note: The number of rows returned by the query is based on the number of camRuns that match the selection +parameters. If an exposure has been processed multiple times, multiple rows will be returned for the same +exposure. Before running the chiptool command duplicates are ignored so the number of runs queued could be +less than the requested value. + +Optional. The default value is 100 exposures. + +=head1 RUN OPTIONS + +The following options may be used to control the parameters of the chipRuns. All are RUN OPTIONS are optional + +=item * --label