Index: /trunk/ippScripts/scripts/camera_exp.pl =================================================================== --- /trunk/ippScripts/scripts/camera_exp.pl (revision 12013) +++ /trunk/ippScripts/scripts/camera_exp.pl (revision 12013) @@ -0,0 +1,278 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Statistics::Descriptive; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($exp_tag, $camera, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'exp_tag|e=s' => \$exp_tag, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --exp_tag --camera", + -exitval => 3, +) unless defined $exp_tag + and defined $camera; + +$ipprc->define_camera($camera); + +use constant RECIPE1 => 'PPIMAGE_J1'; # Recipe to use for ppImage to make JPEGs +use constant RECIPE2 => 'PPIMAGE_J2'; # Recipe to use for ppImage to make JPEGs + + +# Look for programs we need +my $missing_tools; +my $camtool = can_run('p3tool') or (warn "Can't find p3tool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); + +# test for addstar and psastro: +my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get list of component files +my $files; # Array of component files +{ + my $command = "$camtool -pendingimfile -exp_tag $exp_tag"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", $exp_tag, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $exp_tag, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $exp_tag, $PS_EXIT_PROG_ERROR); +} + +# Gather the statistics +my ($bg, $bg_stdev, $bg_mean_stdev); # The statistics triplet +{ + my @backgrounds; # Array of backgrounds in each component + my @variances; # Array of variances for each component +# my @ra; # Array of ra errors +# my @dec; # Array of dec errors +# my @zp; # Array of photometric zero points + foreach my $file (@$files) { + &my_die("Unable to find class id", $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{class_id}; + my $class_id = $file->{class_id}; + &my_die("Unable to find bg for class_id=$class_id", $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{bg}; + &my_die("Unable to find bg_mean_stdev for class_id=$class_id", $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{bg_mean_stdev}; +# &my_die("Unable to find sigma_ra for class_id=$class_id", $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{sigma_ra}; +# &my_die("Unable to find sigma_dec for class_id=$class_id", $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{sigma_dec}; +# &my_die("Unable to find zp for class_id=$class_id", $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{zp}; + push @backgrounds, $file->{bg}; + push @variances, $file->{bg_stdev}**2; + } + + { + my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator + $stats->add_data(@backgrounds); + $bg = ($stats->mean() or 'NAN'); + $bg_mean_stdev = ($stats->standard_deviation() or 'NAN'); + } + { + my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator + $stats->add_data(@variances); + $bg_stdev = (sqrt( $stats->mean() ) or 'NAN'); + } +} + +if (defined $workdir) { + $workdir = $ipprc->convert_filename_absolute( $workdir ); +} else { + my $example = ${$files}[0]->{b1_uri}; # Example original root name + my ($vol, $dir, $file) = File::Spec->splitpath( $ipprc->convert_filename_absolute( $example ) ); + $workdir = $dir; +} +system "mkdir -p $workdir" unless -d $workdir; + +# make this a function which generates a specific list file for a specific filename + +# Generate the file list, and get the statistics +my $outputFile = "$exp_tag.p3"; +my $outputRoot = File::Spec->catfile( $workdir, $outputFile ); + +my $list1Name = $outputRoot . '.b1.list'; # Name for the input file list for binning 1 +my $list2Name = $outputRoot . '.b2.list'; # Name for the input file list for binning 2 +my $list3Name = $outputRoot . '.b3.list'; # Name for the input file list for chip astrometry + +my @means; # Array of means +my @stdevs; # Array of stdevs +open my $list1File, '>' . $list1Name; +open my $list2File, '>' . $list2Name; +open my $list3File, '>' . $list3Name; + +print "$list1Name $list2Name $list3Name\n"; + +my $chipObjects; +foreach my $file (@$files) { + # use the b1_uri as OUTPUT root and convert the filenames + # with ipprc->filename: + my $class_id = $file->{class_id}; + my $origRoot = $ipprc->convert_filename_absolute( $file->{b1_uri} ); # Original root name + + # if there is only one chip, we use this name for the input to addstar + $chipObjects = $ipprc->filename("PSASTRO.OUTPUT", $origRoot, $class_id); + print $list1File $ipprc->filename("PPIMAGE.BIN1", $origRoot, $class_id); + print $list2File $ipprc->filename("PPIMAGE.BIN2", $origRoot, $class_id); + print $list3File $chipObjects; + push @means, $file->{bg}; + push @stdevs, $file->{bg_stdev}; +} +close $list1File; +close $list2File; +close $list3File; + +# Output products --- need to synch with the camera configuration! +my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outputRoot); # Binned JPEG #1 +my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2 +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $outputRoot); # MEF psastro output + +unless ($no_op) { + + # run psastro +mosastro on the set of chips + # XXX note that this is wrong if imfiles are cells + if (@$files > 1) { + my $command = "$psastro -list $list3Name $outputRoot +mosastro -chipastro"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psastro: $error_code", $exp_tag, $error_code); + } + &my_die("Unable to find expected output file: $fpaObjects", $exp_tag, $PS_EXIT_PROG_ERROR) unless -f $fpaObjects; + } else { + $fpaObjects = $chipObjects; + } + + # run addstar on either the single chip output or the single fpa output + # XXX this construct requires the user to have a valid .ptolemyrc + # XXX which in turn points at ippconfig/dvo.site + { + my $command = "addstar -D CAMERA $camera $fpaObjects"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $exp_tag, $error_code); + } + } + + # Make the jpeg for binning 1 + { + my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE " . RECIPE1; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $exp_tag, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1", $exp_tag, $PS_EXIT_PROG_ERROR) unless -f $jpeg1; + } + + # Make the jpeg for binning 2 + { + my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE " . RECIPE2; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $exp_tag, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2", $exp_tag, $PS_EXIT_PROG_ERROR) unless -f $jpeg2; + } +} + +# XXX keep the same outroot as the input +# Add the result into the database +$outputRoot = $ipprc->convert_filename_relative($outputRoot); +$jpeg1 = $ipprc->convert_filename_relative($jpeg1); +$jpeg2 = $ipprc->convert_filename_relative($jpeg2); + +unless ($no_update) { + my $command = "$camtool -addprocessedexp -exp_tag $exp_tag -uri UNKNOWN " . + "-recip " . RECIPE1() . "," . RECIPE2() . " -b1_uri $outputRoot " . + "-bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev " . + "-sigma_ra 0.0 -sigma_dec 0.0 -nastro 0 -zp_mean 0.0 -zp_stdev 0.0"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addprocessedexp: $error_code\n"); + exit($error_code); + } + + unlink $list1Name; + unlink $list2Name; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_tag = shift; # Exposure tag + my $exit_code = shift; # Exit code to add + + warn($msg); + if ($exp_tag and not $no_update) { + my $command = "$camtool -addprocessedexp -exp_tag $exp_tag -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /trunk/ippScripts/scripts/chip_imfile.pl =================================================================== --- /trunk/ippScripts/scripts/chip_imfile.pl (revision 12013) +++ /trunk/ippScripts/scripts/chip_imfile.pl (revision 12013) @@ -0,0 +1,186 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use IPC::Cmd qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use Data::Dumper; +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use constant RECIPE => 'PPIMAGE_OBDSFRA'; # Recipe to use + +# Parse the command-line arguments +my ($exp_tag, # Exposure tag + $class_id, # Class Id + $input, # Input FITS file + $camera, # Camera + $dbname, # Database name + $workdir, # Working directory, for output files + $no_update, # Don't update the database? + $no_op, # Don't do any operations? + ); +GetOptions( + 'exp_tag|e=s' => \$exp_tag, + 'class_id|i=s' => \$class_id, + 'uri|u=s' => \$input, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --exp_tag --class_id --uri --camera", + -exitval => 3, +) unless defined $exp_tag + and defined $class_id + and defined $input + and defined $camera; + +$ipprc->define_camera($camera); + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('p2tool') or (warn "Can't find p2tool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} +$ppImage .= " -dbname $dbname" if defined $dbname; + +$input = $ipprc->convert_filename_absolute( $input ); + +if (defined $workdir) { + $workdir = $ipprc->convert_filename_absolute( $workdir ); +} else { + my ($vol, $dir, $file) = File::Spec->splitpath( $input ); + $workdir = $dir; +} + +$workdir = File::Spec->catfile( $workdir, $exp_tag ); +system "mkdir -p $workdir" unless -d $workdir; + +### Output file name --- must match camera configuration! +my $outputFile = "$exp_tag.p2"; +my $outputRoot = File::Spec->catfile( $workdir, $outputFile ); +my $outputImage = $ipprc->filename("PPIMAGE.OUTPUT", $outputRoot, $class_id); +my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outputRoot, $class_id); +my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outputRoot, $class_id); +my $outputStats = $outputRoot . '.' . $class_id . '.stats'; + +my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser +unless ($no_op) { + # Run ppImage + print "outputImage: $outputImage\n"; + print "outputBin1: $outputBin1\n"; + print "outputStats: $outputStats\n"; + + my $command = "$ppImage -file $input $outputRoot -recipe PPIMAGE " . RECIPE . + " -stat $outputStats"; # Command to run ppImage + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $exp_tag, $class_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputImage\n", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputImage; + &my_die("Couldn't find expected output file: $outputBin1\n", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin1; + &my_die("Couldn't find expected output file: $outputBin2\n", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin2; + &my_die("Couldn't find expected output file: $outputStats\n", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStats; + + # Get the statistics on the processed image + my $statsFile; # File handle + open $statsFile, $outputStats or &my_die("Can't open statistics file $outputStats: $!", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents) or + &my_die("Unable to parse metadata config doc", $exp_tag, $class_id, $PS_EXIT_PROG_ERROR); + $stats->parse($metadata) or + &my_die("Unable to find all values in statistics output.\n", $exp_tag, $class_id, $PS_EXIT_PROG_ERROR); +} + +# Add the processed file to the database +$outputImage = $ipprc->convert_filename_relative( $outputImage); +$outputRoot = $ipprc->convert_filename_relative( $outputRoot ); + +my $bg = ($stats->bg_mean() or 'NAN'); +my $bg_stdev = ($stats->bg_stdev() or 'NAN'); +my $bg_mean_stdev = ($stats->bg_mean_stdev() or 'NAN'); + +unless ($no_update) { + # Command to run chiptool + my $command = "$chiptool -addprocessedimfile"; + $command .= " -exp_tag $exp_tag"; + $command .= " -class_id $class_id"; + $command .= " -recip " . RECIPE; + $command .= " -uri $outputImage"; + $command .= " -b1_uri $outputRoot"; + $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform chiptool -addprocessedimfile: $error_code\n"); + exit($error_code); + } + + unlink $outputStats; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_tag = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if ($exp_tag and $class_id and not $no_update) { + my $command = "$chiptool -addprocessedimfile"; + $command .= " -exp_tag $exp_tag"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -uri $outputImage"; + $command .= " -recip " . RECIPE; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /trunk/ippScripts/scripts/diff_skycell.pl =================================================================== --- /trunk/ippScripts/scripts/diff_skycell.pl (revision 12013) +++ /trunk/ippScripts/scripts/diff_skycell.pl (revision 12013) @@ -0,0 +1,213 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use Data::Dumper; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($p5_id, $camera, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'p5_id|d=s' => \$p5_id, # Phase 5 identifier + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --p5_id --camera", + -exitval => 3, +) unless defined $p5_id + and defined $camera; + +$ipprc->define_camera($camera); + +# Look for programs we need +my $missing_tools; +my $difftool = can_run('p5tool') or (warn "Can't find p5tool" and $missing_tools = 1); +my $pois = can_run('pois') or (warn "Can't find pois" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of components for subtraction +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$difftool -inputscfile -p5_id $p5_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -inputscfile: $error_code", $p5_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $p5_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $p5_id, $PS_EXIT_PROG_ERROR); +} + +&my_die("Subtraction list contains more than two elements", $p5_id, $PS_EXIT_SYS_ERROR) unless + scalar @$files == 2; + +# Identify the input and the template +my $input; # Input file +my $template; # Template file +my $skycell_id; # Skycell identifier +foreach my $file (@$files) { + my $uri = $ipprc->convert_filename_absolute( $file->{uri} ); # URI for file + if (defined $file->{template} and $file->{template}) { + $template = $uri; + } else { + $input = $uri; + } + if (defined $skycell_id) { + &my_die("Skycell identifiers don't match", $p5_id, $PS_EXIT_SYS_ERROR) unless + $file->{skycell_id} eq $skycell_id; + } else { + $skycell_id = $file->{skycell_id}; + } +} + +&my_die("Unable to identify template", $p5_id, $PS_EXIT_SYS_ERROR) unless defined $template; +&my_die("Unable to identify input", $p5_id, $PS_EXIT_SYS_ERROR) unless defined $input; + +### Working directory +if (defined $workdir) { + $workdir = $ipprc->convert_filename_absolute( $workdir ); +} else { + my ($vol, $dir, $file) = File::Spec->splitpath( $input ); + $workdir = $dir; +} + +# Get the output filenames +my $outputFile = "$skycell_id.sub.$p5_id"; # Root name +my $outputRoot = File::Spec->catfile( $workdir, $outputFile ); + +my $outputName = $outputRoot . ".fits"; +my $bin1Name = $ipprc->filename("PPIMAGE.BIN1", $outputRoot); +my $bin2Name = $ipprc->filename("PPIMAGE.BIN2", $outputRoot); +my $outputStats = $outputRoot . '.stats'; + +# Perform subtraction +my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser +unless ($no_op) { + my $command = "$pois $template $input $outputName"; # Command to run pois + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $p5_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $p5_id, $PS_EXIT_SYS_ERROR) unless -f $outputName; +# &my_die("Couldn't find expected output file: $bin1Name", $p5_id, $PS_EXIT_SYS_ERROR) unless -f $bin1Name; +# &my_die("Couldn't find expected output file: $bin2Name", $p5_id, $PS_EXIT_SYS_ERROR) unless -f $bin2Name; +# &my_die("Couldn't find expected output file: $outputStats", $p5_id, $PS_EXIT_SYS_ERROR) unless -f $outputStats; + + # Get the statistics on the residual image + if (0) { ### Disabled because pois doesn't output stats yet + my $statsFile; # File handle + open $statsFile, $outputStats or &my_die("Can't open statistics file $outputStats: $!", $p5_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $metadata = $mdcParser->parse(join "", @contents) or + &my_die("Unable to parse metadata config doc", $p5_id, $PS_EXIT_PROG_ERROR); + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $p5_id, $PS_EXIT_PROG_ERROR); + } +} + +# Add the processed file to the database +$outputName = $ipprc->convert_filename_relative( $outputName ); + +my $bg = ($stats->bg_mean() or 'NAN'); +my $bg_stdev = ($stats->bg_stdev() or 'NAN'); + +unless ($no_update) { + + # Add the subtraction result + { + my $command = "$difftool -adddiffscfile -p5_id $p5_id -uri $outputName -b1_uri $outputRoot"; + $command .= " -bg $bg -bg_stdev $bg_stdev"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -adddiffscfile: $error_code", $p5_id, $error_code); + } + +# unlink $outputStats; + } + + # Register the run as completed + { + my $command = "$difftool -updaterun -p5_id $p5_id -state stop"; # Command to run p5tool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform difftool -updaterun: $error_code\n"); + exit($error_code); + } + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $p5_id = shift; # Phase 5 identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if ($p5_id and not $no_update) { + my $command = "$difftool -updaterun -p5_id $p5_id -state stop -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /trunk/ippScripts/scripts/ipp_serial_camera.pl =================================================================== --- /trunk/ippScripts/scripts/ipp_serial_camera.pl (revision 12013) +++ /trunk/ippScripts/scripts/ipp_serial_camera.pl (revision 12013) @@ -0,0 +1,73 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $workdir, # Working directory + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname|d=s' => \$dbname, + 'workdir|w=s' => \$workdir, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $camtool = can_run('p3tool') or (warn "Can't find p3tool" and $missing_tools = 1); +my $phase3 = can_run('phase3.pl') or (warn "Can't find phase3.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Phase 3 exposure processing +my $list; +{ + my $command = "$camtool -pendingexp -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 3 exposure list: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from chiptool.\n"; +} + +foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $camera = $item->{camera}; + + my $command = "$phase3 --exp_tag $exp_tag --camera $camera --dbname $dbname"; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 3 processing on $exp_tag: $error_code\n" if not $success; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /trunk/ippScripts/scripts/ipp_serial_chip.pl =================================================================== --- /trunk/ippScripts/scripts/ipp_serial_chip.pl (revision 12013) +++ /trunk/ippScripts/scripts/ipp_serial_chip.pl (revision 12013) @@ -0,0 +1,75 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $workdir, # Working directory + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname|d=s' => \$dbname, + 'workdir|w=s' => \$workdir, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('p2tool') or (warn "Can't find p2tool" and $missing_tools = 1); +my $phase2 = can_run('phase2.pl') or (warn "Can't find phase2.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Phase 2 imfile processing +my $list; +{ + my $command = "$chiptool -pendingimfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 2 imfile list: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from chiptool.\n"; +} + +foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $camera = $item->{camera}; + my $class_id = $item->{class_id}; + my $uri = $item->{uri}; + + my $command = "$phase2 --exp_tag $exp_tag --class_id $class_id --uri $uri --dbname $dbname --camera $camera"; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 2 processing on $exp_tag $class_id: $error_code\n" if not $success; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: unk/ippScripts/scripts/ipp_serial_phase0.pl =================================================================== --- /trunk/ippScripts/scripts/ipp_serial_phase0.pl (revision 12012) +++ (revision ) @@ -1,90 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -use IPC::Cmd qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::List qw( parse_md_list ); -use Data::Dumper; - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -my ($dbname, # Database name to use - $workdir, # Working directory - ); -GetOptions( - 'dbname|d=s' => \$dbname, - 'workdir|w=s' => \$workdir, -) or pod2usage( 2 ); - -pod2usage( - -msg => "Required options: --dbname", - -exitval => 3, - ) unless defined $dbname; - -my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser - -# Look for programs we need -my $missing_tools; -my $p0tool = can_run('p0tool') or (warn "Can't find p0tool" and $missing_tools = 1); -my $phase0_imfile = can_run('phase0_imfile.pl') or (warn "Can't find phase0_imfile.pl" and $missing_tools = 1); -my $phase0_exp = can_run('phase0_exp.pl') or (warn "Can't find phase0_exp.pl" and $missing_tools = 1); -die "Can't find required tools.\n" if $missing_tools; - -# Phase 0 imfile processing -my $list; -{ - my $command = "$p0tool -pendingimfile -dbname $dbname"; # Command to run - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run( command => $command, verbose => 1 ); - die "Unable to get phase 0 imfile list: $error_code\n" if not $success; - $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or - die "Unable to parse output from p0tool.\n"; -} - -foreach my $item (@$list) { - my $exp_tag = $item->{exp_tag}; - my $class = $item->{class}; - my $class_id = $item->{class_id}; - my $uri = $item->{uri}; - - my $command = "$phase0_imfile --exp_tag $exp_tag --class $class --class_id $class_id --uri $uri --dbname $dbname"; - $command .= " --workdir $workdir" if defined $workdir; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run( command => $command, verbose => 1 ); - die "Unable to do phase 0 imfile processing on $exp_tag $class_id: $error_code\n" if not $success; -} - -# Phase 0 exposure processing -{ - my $command = "$p0tool -pendingexp -dbname $dbname"; # Command to run - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run( command => $command, verbose => 1 ); - die "Unable to get phase 0 exposure list: $error_code\n" if not $success; - $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or - die "Unable to parse output from p0tool.\n"; -} - -foreach my $item (@$list) { - my $exp_tag = $item->{exp_tag}; - - my $command = "$phase0_exp --exp_tag $exp_tag --dbname $dbname"; - $command .= " --workdir $workdir" if defined $workdir; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run( command => $command, verbose => 1 ); - die "Unable to do phase 0 exposure processing on $exp_tag: $error_code\n" if not $success; -} - -END { - my $status = $?; - system("sync") == 0 - or die "failed to execute sync: $!" ; - $? = $status; -} - - -__END__ - - Index: unk/ippScripts/scripts/ipp_serial_phase2.pl =================================================================== --- /trunk/ippScripts/scripts/ipp_serial_phase2.pl (revision 12012) +++ (revision ) @@ -1,75 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -use IPC::Cmd qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::List qw( parse_md_list ); -use Data::Dumper; - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -my ($dbname, # Database name to use - $workdir, # Working directory - $no_op, # No operations? - $no_update, # No updating? - ); -GetOptions( - 'dbname|d=s' => \$dbname, - 'workdir|w=s' => \$workdir, - 'no-op' => \$no_op, - 'no-update' => \$no_update, -) or pod2usage( 2 ); - -pod2usage( - -msg => "Required options: --dbname", - -exitval => 3, - ) unless defined $dbname; - -my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser - -# Look for programs we need -my $missing_tools; -my $p2tool = can_run('p2tool') or (warn "Can't find p2tool" and $missing_tools = 1); -my $phase2 = can_run('phase2.pl') or (warn "Can't find phase2.pl" and $missing_tools = 1); -die "Can't find required tools.\n" if $missing_tools; - -# Phase 2 imfile processing -my $list; -{ - my $command = "$p2tool -pendingimfile -dbname $dbname"; # Command to run - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run( command => $command, verbose => 1 ); - die "Unable to get phase 2 imfile list: $error_code\n" if not $success; - $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or - die "Unable to parse output from p2tool.\n"; -} - -foreach my $item (@$list) { - my $exp_tag = $item->{exp_tag}; - my $camera = $item->{camera}; - my $class_id = $item->{class_id}; - my $uri = $item->{uri}; - - my $command = "$phase2 --exp_tag $exp_tag --class_id $class_id --uri $uri --dbname $dbname --camera $camera"; - $command .= " --no-op" if defined $no_op; - $command .= " --no-update" if defined $no_update; - $command .= " --workdir $workdir" if defined $workdir; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run( command => $command, verbose => 1 ); - die "Unable to do phase 2 processing on $exp_tag $class_id: $error_code\n" if not $success; -} - -END { - my $status = $?; - system("sync") == 0 - or die "failed to execute sync: $!" ; - $? = $status; -} - - -__END__ - - Index: unk/ippScripts/scripts/ipp_serial_phase3.pl =================================================================== --- /trunk/ippScripts/scripts/ipp_serial_phase3.pl (revision 12012) +++ (revision ) @@ -1,73 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -use IPC::Cmd qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::List qw( parse_md_list ); -use Data::Dumper; - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -my ($dbname, # Database name to use - $workdir, # Working directory - $no_op, # No operations? - $no_update, # No updating? - ); -GetOptions( - 'dbname|d=s' => \$dbname, - 'workdir|w=s' => \$workdir, - 'no-op' => \$no_op, - 'no-update' => \$no_update, -) or pod2usage( 2 ); - -pod2usage( - -msg => "Required options: --dbname", - -exitval => 3, - ) unless defined $dbname; - -my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser - -# Look for programs we need -my $missing_tools; -my $p3tool = can_run('p3tool') or (warn "Can't find p3tool" and $missing_tools = 1); -my $phase3 = can_run('phase3.pl') or (warn "Can't find phase3.pl" and $missing_tools = 1); -die "Can't find required tools.\n" if $missing_tools; - -# Phase 3 exposure processing -my $list; -{ - my $command = "$p3tool -pendingexp -dbname $dbname"; # Command to run - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run( command => $command, verbose => 1 ); - die "Unable to get phase 3 exposure list: $error_code\n" if not $success; - $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or - die "Unable to parse output from p2tool.\n"; -} - -foreach my $item (@$list) { - my $exp_tag = $item->{exp_tag}; - my $camera = $item->{camera}; - - my $command = "$phase3 --exp_tag $exp_tag --camera $camera --dbname $dbname"; - $command .= " --no-op" if defined $no_op; - $command .= " --no-update" if defined $no_update; - $command .= " --workdir $workdir" if defined $workdir; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run( command => $command, verbose => 1 ); - die "Unable to do phase 3 processing on $exp_tag: $error_code\n" if not $success; -} - -END { - my $status = $?; - system("sync") == 0 - or die "failed to execute sync: $!" ; - $? = $status; -} - - -__END__ - - Index: /trunk/ippScripts/scripts/ipp_serial_register.pl =================================================================== --- /trunk/ippScripts/scripts/ipp_serial_register.pl (revision 12013) +++ /trunk/ippScripts/scripts/ipp_serial_register.pl (revision 12013) @@ -0,0 +1,90 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $workdir, # Working directory + ); +GetOptions( + 'dbname|d=s' => \$dbname, + 'workdir|w=s' => \$workdir, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('p0tool') or (warn "Can't find p0tool" and $missing_tools = 1); +my $phase0_imfile = can_run('phase0_imfile.pl') or (warn "Can't find phase0_imfile.pl" and $missing_tools = 1); +my $phase0_exp = can_run('phase0_exp.pl') or (warn "Can't find phase0_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Phase 0 imfile processing +my $list; +{ + my $command = "$regtool -pendingimfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 0 imfile list: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from regtool.\n"; +} + +foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $class = $item->{class}; + my $class_id = $item->{class_id}; + my $uri = $item->{uri}; + + my $command = "$phase0_imfile --exp_tag $exp_tag --class $class --class_id $class_id --uri $uri --dbname $dbname"; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 0 imfile processing on $exp_tag $class_id: $error_code\n" if not $success; +} + +# Phase 0 exposure processing +{ + my $command = "$regtool -pendingexp -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 0 exposure list: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from regtool.\n"; +} + +foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + + my $command = "$phase0_exp --exp_tag $exp_tag --dbname $dbname"; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 0 exposure processing on $exp_tag: $error_code\n" if not $success; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: unk/ippScripts/scripts/phase0_exp.pl =================================================================== --- /trunk/ippScripts/scripts/phase0_exp.pl (revision 12012) +++ (revision ) @@ -1,267 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -## report the program and machine -use Sys::Hostname; -my $host = hostname(); -print "\n\n"; -print "Starting script $0 on $host\n\n"; - -use Cache::File; -use Storable qw( freeze thaw ); -use File::Basename qw( basename ); -use IPC::Cmd qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::List qw( parse_md_list ); -use Statistics::Descriptive; - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -use PS::IPP::Config qw( - $PS_EXIT_SUCCESS - $PS_EXIT_UNKNOWN_ERROR - $PS_EXIT_SYS_ERROR - $PS_EXIT_CONFIG_ERROR - $PS_EXIT_PROG_ERROR - $PS_EXIT_DATA_ERROR - $PS_EXIT_TIMEOUT_ERROR - ); - -my ($cache, $exptag, $workdir, $dbname, $no_update); - -GetOptions( - 'caches' => \$cache, - 'exp_tag|e=s' => \$exptag, - 'workdir|w=s' => \$workdir, # Working directory for output files - 'dbname|d=s' => \$dbname, # Database name - 'no-update' => \$no_update -) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( - -msg => "Required options: --exp_tag", - -exitval => 3, -) unless defined $exptag; - -# Define setup -use constant TYPE => "exp_type"; # Observation type keyword -use constant DETRENDS => [ "bias", "zero", "dark", "shutter", "flat", "fringe", "domeflat", "skyflat" ]; # Observation types to mark as detrend -use constant DETREND_FLAG => "-detrend"; # Flag to use to mark detrend exposure - -use constant PHASE0_URI => 'uri'; # Key for the URI -use constant PHASE0_CLASSID => 'class_id'; # Key for the class id -use constant PHASE0_BG => 'bg'; # Key for the background -use constant PHASE0_BG_STDEV => 'bg_stdev'; # Key for the background standard deviation -use constant PHASE0_BG_MEAN_STDEV => 'bg_mean_stdev'; # Key for the mean of the background standard deviation - -# These values should be constant for all components -use constant CONSTANTS => [ - "object", # Object - "exp_type", # Exposure type - "filter", # Filter used - "airmass", # Airmass - "ra", # Right ascension - "decl", # Declination - "posang", # Position angle - "alt", # Altitude - "az", # Azimuth - "ccd_temp", # CCD temperature - "dateobs", # Time of exposure - ]; - -# These values may vary across components; we will take the average -use constant VARIABLES => [ - "exp_time", # Exposure time - ]; - - -# Look for commands we need -my $missing_tools; -my $p0tool = can_run('p0tool') - or (warn "can't find p0tool" and $missing_tools = 1); -my $ppStats = can_run('ppStats') - or (warn "can't find ppStats" and $missing_tools = 1); - -if ($missing_tools) { - warn ("Can't find required tools"); - exit($PS_EXIT_CONFIG_ERROR); -} - -# setup cache interface - -my $c = Cache::File->new( - cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)), - default_expires => '7200 sec', -); - -my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - -# Get the list of imfiles -my $imfiles; -{ - my $command = "$p0tool -processedimfile -exp_tag $exptag"; - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - cache_run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn ("Unable to perform p0tool on exposure id $exptag: $error_code"); - exit ($error_code); - } - - my $metadata = $mdcParser->parse(join "", @$stdout_buf); - unless ($metadata) { - warn ("Unable to parse metadata config doc"); - &my_die ($exptag, $PS_EXIT_PROG_ERROR); - } - $imfiles = parse_md_list($metadata); # Data for imfiles -} - -# Process the values -my %values; # Values to update -foreach my $variable (@{VARIABLES()}) { - $values{$variable} = []; -} -my @backgrounds; # Array of backgrounds -my @variances; # Array of background standard deviations\ -my $obsType; # Observation type - -foreach my $imfile (@$imfiles) { - foreach my $constant (@{CONSTANTS()}) { - my $value = get_value($imfile, $constant); # Value for imfile - if (not defined $values{$constant}) { - $values{$constant} = $value; - } - if ($values{$constant} ne $value) { - warn ("Value of $constant for $imfile->{PHASE0_CLASSID} doesn't match previous value"); - &my_die ($exptag, $PS_EXIT_PROG_ERROR); - } - } - - foreach my $variable (@{VARIABLES()}) { - my $value = get_value($imfile, $variable); # Value for imfile - my $array = $values{$variable}; # Array of data - push @$array, $value; - } - - my $bg = get_value($imfile, PHASE0_BG()); - push @backgrounds, $bg; - - my $stdev = get_value($imfile, PHASE0_BG_MEAN_STDEV()); - push @variances, $stdev**2; -} - -if (0) { -# XXX for a test, randomly declare a failure and return to pantasks -my $rnd = rand(1); -if ($rnd > 0.5) { - warn ("random failure"); - &my_die ($exptag, $PS_EXIT_DATA_ERROR); -} -} - -# Output results to the database -unless ($no_update) { - my @command; - push @command, $p0tool, '-addprocessedexp', '-exp_tag', $exptag; # Command to execute to update exposure parameters - - # Add the values of interest - foreach my $constant (@{CONSTANTS()}) { - push @command, ( '-' . $constant ), $values{$constant}; - } - foreach my $variable (@{VARIABLES()}) { - my $array = $values{$variable}; # Array of values - my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator - $stats->add_data(@$array); - push @command, ( '-' . $variable ), $stats->mean(); - } - - # Add the statistics - { - my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator - $stats->add_data(@backgrounds); - push @command, ( '-' . PHASE0_BG() ), $stats->mean(); - push @command, ( '-' . PHASE0_BG_MEAN_STDEV() ), ( $stats->standard_deviation() || 0 ) - } - { - my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator - $stats->add_data(@variances); - push @command, ( '-' . PHASE0_BG_STDEV() ), sqrt( $stats->mean() ); - } - - # Add the detrend flag - foreach my $detrendType (@{DETRENDS()}) { - if (lc($values{TYPE()}) =~ /$detrendType/) { - push @command, DETREND_FLAG; - last; - } - } - - # Quote arguments with whitespace - for (my $i = 0; $i < scalar @command; $i++) { - if ($command[$i] =~ /\s/) { - $command[$i] = "\'$command[$i]\'"; - } - } - - push @command, "-dbname", $dbname if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - cache_run(command => \@command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn ("Unable to run phase0 update for $exptag: $error_code"); - exit($error_code); - } -} - -### Pau. - -# Get the value for a particular imfile, along with a strong check for its existence -sub get_value { - my $imfile = shift; # The hash of values - my $name = shift; # The name of the value to check - - my $source = $imfile->{PHASE0_CLASSID()}; # Where it comes from - - unless (defined $imfile->{$name}) { - warn ("Couldn't find value of $name for class_id=$source"); - &my_die ($exptag, $PS_EXIT_PROG_ERROR); - } - return $imfile->{$name}; -} - -sub cache_run -{ - my %p = @_; - - my $cmd_output = $c->get($p{command}) if $cache; - if (defined $cmd_output) { - return @{thaw $cmd_output}; - } else { - my @output = run(%p); - $c->set($p{command}, freeze \@output) if $cache; - return @output; - } -} - -sub my_die -{ - my $exp_tag = $_[0]; - my $exit_code = $_[1]; - if ($exp_tag and not $no_update) { - my $command = "$p0tool -addprocessedexp -exp_tag $exp_tag -code $exit_code"; - $command .= " -dbname $dbname" if defined $dbname; - system($command); - } - exit $exit_code; -} - -END { - my $exit = $?; - system("sync") == 0 or die "failed to execute sync: $!"; - $? = $exit; -} Index: unk/ippScripts/scripts/phase0_imfile.pl =================================================================== --- /trunk/ippScripts/scripts/phase0_imfile.pl (revision 12012) +++ (revision ) @@ -1,250 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -## report the program and machine -use Sys::Hostname; -my $host = hostname(); -print "\n\n"; -print "Starting script $0 on $host\n\n"; - -use vars qw( $VERSION ); -$VERSION = '0.01'; - -use Cache::File; -use Storable qw(freeze thaw); -use File::Basename qw( basename); -use IPC::Cmd qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; -use Data::Dumper; - -use PS::IPP::Config qw( - $PS_EXIT_SUCCESS - $PS_EXIT_UNKNOWN_ERROR - $PS_EXIT_SYS_ERROR - $PS_EXIT_CONFIG_ERROR - $PS_EXIT_PROG_ERROR - $PS_EXIT_DATA_ERROR - $PS_EXIT_TIMEOUT_ERROR - ); - -my $ipprc = PS::IPP::Config->new(); # IPP configuration -use File::Spec; - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -my ($cache, $exp_tag, $class_id, $uri, $workdir, $dbname, $no_update); - -# print "args: @ARGV\n"; -# for (my $i = 0; $i < @ARGV; $i++) { -# print "arg $i: ...$ARGV[$i]...\n"; -# } - -GetOptions( - 'caches' => \$cache, - 'exp_tag|e=s' => \$exp_tag, - 'class_id|i=s' => \$class_id, - 'uri|u=s' => \$uri, - 'workdir|w=s' => \$workdir, # Working directory for output files - 'dbname|d=s' => \$dbname,# Database name - 'no-update' => \$no_update -) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( - -msg => "Required options: --exp_tag --class_id --uri", - -exitval => 3, -) unless defined $exp_tag - and defined $class_id - and defined $uri; - -my $RECIPE = "PPSTATS_PHASE0"; # Recipe to use for ppStats -# use constant RECIPE => "PPSTATS_PHASE0"; # Recipe to use for ppStats - -# These values should be constant for all components -# The key is the name in the ppStats output; the value is the switch for "phase0 -updateexp" -use constant CONSTANTS => { - "FPA.OBJECT" => "-object", # Object - "FPA.OBSTYPE" => "-exp_type", # Exposure type - "FPA.FILTER" => "-filter", # Filter used - "FPA.AIRMASS" => "-airmass", # Airmass - "FPA.RA" => "-ra", # Right ascension - "FPA.DEC" => "-decl", # Declination - "FPA.ALT" => "-alt", # Altitude - "FPA.AZ" => "-az", # Azimuth - "FPA.POSANGLE" => "-posang", # Position angle - "FPA.TIME" => "-dateobs", # Date of observation (UTC) - }; - -# These values may vary across components; we will take the average -# The key is the name in the ppStats output; the value is the switch for "phase0 -updateexp" -use constant VARIABLES => { - "CHIP.TEMP" => "-ccd_temp", # CCD temperature - "CELL.EXPOSURE" => "-exp_time" # Exposure time - }; - - -# Switches for p0tool -use constant P0TOOL_MODE => '-addprocessedimfile'; # Mode for p0tool -use constant P0TOOL_EXPTAG => '-exp_tag'; # Switch to specify the exposure id -use constant P0TOOL_CLASSID => '-class_id'; # Switch to specify the class id -use constant P0TOOL_BG_MEAN => '-bg'; # Switch to add the background -use constant P0TOOL_BG_STDEV => '-bg_stdev'; # Switch to add the bg stdev -use constant P0TOOL_BG_MEAN_STDEV => '-bg_mean_stdev'; # Switch to add the bg mean stdev - -# Look for programs we need -my $missing_tools; -my $p0tool = can_run('p0tool') - or (warn "Can't find p0tool" and $missing_tools = 1); -my $ppStats = can_run('ppStats') - or (warn "Can't find ppStats" and $missing_tools = 1); - -if ($missing_tools) { - warn ("Can't find required tools"); - exit($PS_EXIT_CONFIG_ERROR); -} - -# setup cache interface - -my $c = Cache::File->new( - cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)), - default_expires => '7200 sec', -); - -# Resolve the input URI -$uri = $ipprc->convert_filename_absolute($uri); - -# Run ppStats on the input file -my $stats; -{ - my $command = "$ppStats $uri -recipe PPSTATS $RECIPE"; # Command to run ppStats - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - cache_run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn ("Unable to perform ppStats on exposure id $exp_tag: $error_code"); - &my_die ($exp_tag, $class_id, $error_code); - } - - # Parse the output - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - my $metadata = $mdcParser->parse(join "", @$stdout_buf); # XXX is this join necessary? - unless ($metadata) { - warn ("Unable to parse metadata config doc"); - &my_die ($exp_tag, $class_id, $PS_EXIT_PROG_ERROR); - } - my @constants = keys %{CONSTANTS()}; # List of constants to parse out - my @variables = keys %{VARIABLES()}; # List of variables to parse out - $stats = PS::IPP::Metadata::Stats->new(\@constants, \@variables); # Stats parser - unless ($stats->parse($metadata)) { - warn ("Unable to find all values"); - &my_die ($exp_tag, $class_id, $PS_EXIT_PROG_ERROR); - # XXX is this a programming or a config error? - } - - if (0) { - # XXX for a test, randomly declare a failure and return to pantasks - my $rnd = rand(1); - if ($rnd > 0.5) { - warn ("random failure"); - &my_die ($exp_tag, $class_id, $PS_EXIT_DATA_ERROR); - } -} -} - -# Push the results into the database -unless ($no_update) { - my @command; - push @command, $p0tool, P0TOOL_MODE(), P0TOOL_EXPTAG(), $exp_tag, P0TOOL_CLASSID(), $class_id; # Command to run p0tool - - foreach my $constant (keys %{CONSTANTS()}) { - push @command, CONSTANTS->{$constant}, ($stats->data($constant))->{value}; - - } - foreach my $variable (keys %{VARIABLES()}) { - # Just use the mean value - push @command, VARIABLES->{$variable}, ($stats->data($variable))->{mean}; - } - - push @command, P0TOOL_BG_MEAN(); - if (defined $stats->bg_mean()) { - push @command, $stats->bg_mean(); - } else { - push @command, "NAN"; - } - push @command, P0TOOL_BG_STDEV(); - if (defined($stats->bg_stdev())) { - push @command, $stats->bg_stdev(); - } else { - # Will not be defined if there is only a single imfile - push @command, 0; - } - push @command, P0TOOL_BG_MEAN_STDEV(); - if (defined $stats->bg_mean_stdev()) { - push @command, $stats->bg_mean_stdev(); - } else { - push @command, "0.0"; - } - - # Quote arguments with whitespace - for (my $i = 0; $i < scalar @command; $i++) { - if ($command[$i] =~ /\s/) { - $command[$i] = "\'$command[$i]\'"; - } - } - - push @command, "-dbname", $dbname if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => \@command, verbose => 1); - - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - # XXX if we can't run -addprocessedimfile, we can't actually set the - # error code. - # XXX if this is not a database error, it is probably a - # programming error (in p0tool or the passed args) - warn ("Unable to perform p0tool -addprocessedimfile: $error_code"); - exit($error_code); - } -} - -sub cache_run -{ - my %p = @_; - - my $cmd_output = $c->get($p{command}) if $cache; - if (defined $cmd_output) { - return @{thaw $cmd_output}; - } else { - my @output = run(%p); - $c->set($p{command}, freeze \@output) if $cache; - return @output; - } -} - -sub my_die -{ - my $exp_tag = $_[0]; - my $class_id = $_[1]; - my $exit_code = $_[2]; - if ($exp_tag && $class_id and not $no_update) { - my $command = "$p0tool -addprocessedimfile -exp_tag $exp_tag -class_id $class_id -code $exit_code"; - $command .= " -dbname $dbname" if defined $dbname; - system ($command); - } - exit $exit_code; -} - -# Pau. - -END { - my $exit = $?; - system("sync") == 0 or die "failed to execute sync: $!"; - $? = $exit; -} - -__END__ Index: unk/ippScripts/scripts/phase2.pl =================================================================== --- /trunk/ippScripts/scripts/phase2.pl (revision 12012) +++ (revision ) @@ -1,186 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -## report the program and machine -use Sys::Hostname; -my $host = hostname(); -print "\n\n"; -print "Starting script $0 on $host\n\n"; - -use IPC::Cmd qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; -use Data::Dumper; -use PS::IPP::Config qw( - $PS_EXIT_SUCCESS - $PS_EXIT_UNKNOWN_ERROR - $PS_EXIT_SYS_ERROR - $PS_EXIT_CONFIG_ERROR - $PS_EXIT_PROG_ERROR - $PS_EXIT_DATA_ERROR - $PS_EXIT_TIMEOUT_ERROR - ); -my $ipprc = PS::IPP::Config->new(); # IPP configuration -use File::Spec; - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -use constant RECIPE => 'PPIMAGE_OBDSFRA'; # Recipe to use - -# Parse the command-line arguments -my ($exp_tag, # Exposure tag - $class_id, # Class Id - $input, # Input FITS file - $camera, # Camera - $dbname, # Database name - $workdir, # Working directory, for output files - $no_update, # Don't update the database? - $no_op, # Don't do any operations? - ); -GetOptions( - 'exp_tag|e=s' => \$exp_tag, - 'class_id|i=s' => \$class_id, - 'uri|u=s' => \$input, - 'camera|c=s' => \$camera, - 'dbname|d=s' => \$dbname, # Database name - 'workdir|w=s' => \$workdir, - 'no-update' => \$no_update, - 'no-op' => \$no_op, -) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( - -msg => "Required options: --exp_tag --class_id --uri --camera", - -exitval => 3, -) unless defined $exp_tag - and defined $class_id - and defined $input - and defined $camera; - -$ipprc->define_camera($camera); - -# Look for programs we need -my $missing_tools; -my $p2tool = can_run('p2tool') or (warn "Can't find p2tool" and $missing_tools = 1); -my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); -if ($missing_tools) { - warn("Can't find required tools."); - exit($PS_EXIT_CONFIG_ERROR); -} -$ppImage .= " -dbname $dbname" if defined $dbname; - -$input = $ipprc->convert_filename_absolute( $input ); - -if (defined $workdir) { - $workdir = $ipprc->convert_filename_absolute( $workdir ); -} else { - my ($vol, $dir, $file) = File::Spec->splitpath( $input ); - $workdir = $dir; -} - -$workdir = File::Spec->catfile( $workdir, $exp_tag ); -system "mkdir -p $workdir" unless -d $workdir; - -### Output file name --- must match camera configuration! -my $outputFile = "$exp_tag.p2"; -my $outputRoot = File::Spec->catfile( $workdir, $outputFile ); -my $outputImage = $ipprc->filename("PPIMAGE.OUTPUT", $outputRoot, $class_id); -my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outputRoot, $class_id); -my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outputRoot, $class_id); -my $outputStats = $outputRoot . '.' . $class_id . '.stats'; - -my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser -unless ($no_op) { - # Run ppImage - print "outputImage: $outputImage\n"; - print "outputBin1: $outputBin1\n"; - print "outputStats: $outputStats\n"; - - my $command = "$ppImage -file $input $outputRoot -recipe PPIMAGE " . RECIPE . - " -stat $outputStats"; # Command to run ppImage - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppImage: $error_code", $exp_tag, $class_id, $error_code); - } - &my_die("Couldn't find expected output file: $outputImage\n", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputImage; - &my_die("Couldn't find expected output file: $outputBin1\n", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin1; - &my_die("Couldn't find expected output file: $outputBin2\n", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin2; - &my_die("Couldn't find expected output file: $outputStats\n", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStats; - - # Get the statistics on the processed image - my $statsFile; # File handle - open $statsFile, $outputStats or &my_die("Can't open statistics file $outputStats: $!", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR); - my @contents = <$statsFile>; # Contents of file - close $statsFile; - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - my $metadata = $mdcParser->parse(join "", @contents) or - &my_die("Unable to parse metadata config doc", $exp_tag, $class_id, $PS_EXIT_PROG_ERROR); - $stats->parse($metadata) or - &my_die("Unable to find all values in statistics output.\n", $exp_tag, $class_id, $PS_EXIT_PROG_ERROR); -} - -# Add the processed file to the database -$outputImage = $ipprc->convert_filename_relative( $outputImage); -$outputRoot = $ipprc->convert_filename_relative( $outputRoot ); - -my $bg = ($stats->bg_mean() or 'NAN'); -my $bg_stdev = ($stats->bg_stdev() or 'NAN'); -my $bg_mean_stdev = ($stats->bg_mean_stdev() or 'NAN'); - -unless ($no_update) { - # Command to run p2tool - my $command = "$p2tool -addprocessedimfile"; - $command .= " -exp_tag $exp_tag"; - $command .= " -class_id $class_id"; - $command .= " -recip " . RECIPE; - $command .= " -uri $outputImage"; - $command .= " -b1_uri $outputRoot"; - $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to perform p2tool -addprocessedimfile: $error_code\n"); - exit($error_code); - } - - unlink $outputStats; -} - - -sub my_die -{ - my $msg = shift; # Warning message on die - my $exp_tag = shift; # Exposure tag - my $class_id = shift; # Class identifier - my $exit_code = shift; # Exit code to add - - warn($msg); - if ($exp_tag and $class_id and not $no_update) { - my $command = "$p2tool -addprocessedimfile"; - $command .= " -exp_tag $exp_tag"; - $command .= " -class_id $class_id"; - $command .= " -code $exit_code"; - $command .= " -uri $outputImage"; - $command .= " -recip " . RECIPE; - $command .= " -dbname $dbname" if defined $dbname; - system ($command); - } - exit $exit_code; -} - -END { - my $status = $?; - system("sync") == 0 - or die "failed to execute sync: $!" ; - $? = $status; -} - -__END__ Index: unk/ippScripts/scripts/phase3.pl =================================================================== --- /trunk/ippScripts/scripts/phase3.pl (revision 12012) +++ (revision ) @@ -1,278 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -## report the program and machine -use Sys::Hostname; -my $host = hostname(); -print "\n\n"; -print "Starting script $0 on $host\n\n"; - -use vars qw( $VERSION ); -$VERSION = '0.01'; - -use IPC::Cmd qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::List qw( parse_md_list ); -use Statistics::Descriptive; - -use PS::IPP::Config qw( - $PS_EXIT_SUCCESS - $PS_EXIT_UNKNOWN_ERROR - $PS_EXIT_SYS_ERROR - $PS_EXIT_CONFIG_ERROR - $PS_EXIT_PROG_ERROR - $PS_EXIT_DATA_ERROR - $PS_EXIT_TIMEOUT_ERROR - ); -my $ipprc = PS::IPP::Config->new(); # IPP configuration -use File::Spec; - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -my ($exp_tag, $camera, $dbname, $workdir, $no_update, $no_op); -GetOptions( - 'exp_tag|e=s' => \$exp_tag, - 'camera|c=s' => \$camera, - 'dbname|d=s' => \$dbname, # Database name - 'workdir|w=s' => \$workdir, - 'no-update' => \$no_update, - 'no-op' => \$no_op, -) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( - -msg => "Required options: --exp_tag --camera", - -exitval => 3, -) unless defined $exp_tag - and defined $camera; - -$ipprc->define_camera($camera); - -use constant RECIPE1 => 'PPIMAGE_J1'; # Recipe to use for ppImage to make JPEGs -use constant RECIPE2 => 'PPIMAGE_J2'; # Recipe to use for ppImage to make JPEGs - - -# Look for programs we need -my $missing_tools; -my $p3tool = can_run('p3tool') or (warn "Can't find p3tool" and $missing_tools = 1); -my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); - -# test for addstar and psastro: -my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); -my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); - -if ($missing_tools) { - warn("Can't find required tools."); - exit($PS_EXIT_CONFIG_ERROR); -} - -my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - -# Get list of component files -my $files; # Array of component files -{ - my $command = "$p3tool -pendingimfile -exp_tag $exp_tag"; # Command to run - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform p3tool: $error_code", $exp_tag, $error_code); - } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $exp_tag, $PS_EXIT_PROG_ERROR); - $files = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $exp_tag, $PS_EXIT_PROG_ERROR); -} - -# Gather the statistics -my ($bg, $bg_stdev, $bg_mean_stdev); # The statistics triplet -{ - my @backgrounds; # Array of backgrounds in each component - my @variances; # Array of variances for each component -# my @ra; # Array of ra errors -# my @dec; # Array of dec errors -# my @zp; # Array of photometric zero points - foreach my $file (@$files) { - &my_die("Unable to find class id", $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{class_id}; - my $class_id = $file->{class_id}; - &my_die("Unable to find bg for class_id=$class_id", $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{bg}; - &my_die("Unable to find bg_mean_stdev for class_id=$class_id", $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{bg_mean_stdev}; -# &my_die("Unable to find sigma_ra for class_id=$class_id", $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{sigma_ra}; -# &my_die("Unable to find sigma_dec for class_id=$class_id", $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{sigma_dec}; -# &my_die("Unable to find zp for class_id=$class_id", $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{zp}; - push @backgrounds, $file->{bg}; - push @variances, $file->{bg_stdev}**2; - } - - { - my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator - $stats->add_data(@backgrounds); - $bg = ($stats->mean() or 'NAN'); - $bg_mean_stdev = ($stats->standard_deviation() or 'NAN'); - } - { - my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator - $stats->add_data(@variances); - $bg_stdev = (sqrt( $stats->mean() ) or 'NAN'); - } -} - -if (defined $workdir) { - $workdir = $ipprc->convert_filename_absolute( $workdir ); -} else { - my $example = ${$files}[0]->{b1_uri}; # Example original root name - my ($vol, $dir, $file) = File::Spec->splitpath( $ipprc->convert_filename_absolute( $example ) ); - $workdir = $dir; -} -system "mkdir -p $workdir" unless -d $workdir; - -# make this a function which generates a specific list file for a specific filename - -# Generate the file list, and get the statistics -my $outputFile = "$exp_tag.p3"; -my $outputRoot = File::Spec->catfile( $workdir, $outputFile ); - -my $list1Name = $outputRoot . '.b1.list'; # Name for the input file list for binning 1 -my $list2Name = $outputRoot . '.b2.list'; # Name for the input file list for binning 2 -my $list3Name = $outputRoot . '.b3.list'; # Name for the input file list for chip astrometry - -my @means; # Array of means -my @stdevs; # Array of stdevs -open my $list1File, '>' . $list1Name; -open my $list2File, '>' . $list2Name; -open my $list3File, '>' . $list3Name; - -print "$list1Name $list2Name $list3Name\n"; - -my $chipObjects; -foreach my $file (@$files) { - # use the b1_uri as OUTPUT root and convert the filenames - # with ipprc->filename: - my $class_id = $file->{class_id}; - my $origRoot = $ipprc->convert_filename_absolute( $file->{b1_uri} ); # Original root name - - # if there is only one chip, we use this name for the input to addstar - $chipObjects = $ipprc->filename("PSASTRO.OUTPUT", $origRoot, $class_id); - print $list1File $ipprc->filename("PPIMAGE.BIN1", $origRoot, $class_id); - print $list2File $ipprc->filename("PPIMAGE.BIN2", $origRoot, $class_id); - print $list3File $chipObjects; - push @means, $file->{bg}; - push @stdevs, $file->{bg_stdev}; -} -close $list1File; -close $list2File; -close $list3File; - -# Output products --- need to synch with the camera configuration! -my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outputRoot); # Binned JPEG #1 -my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2 -my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $outputRoot); # MEF psastro output - -unless ($no_op) { - - # run psastro +mosastro on the set of chips - # XXX note that this is wrong if imfiles are cells - if (@$files > 1) { - my $command = "$psastro -list $list3Name $outputRoot +mosastro -chipastro"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform psastro: $error_code", $exp_tag, $error_code); - } - &my_die("Unable to find expected output file: $fpaObjects", $exp_tag, $PS_EXIT_PROG_ERROR) unless -f $fpaObjects; - } else { - $fpaObjects = $chipObjects; - } - - # run addstar on either the single chip output or the single fpa output - # XXX this construct requires the user to have a valid .ptolemyrc - # XXX which in turn points at ippconfig/dvo.site - { - my $command = "addstar -D CAMERA $camera $fpaObjects"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform addstar: $error_code", $exp_tag, $error_code); - } - } - - # Make the jpeg for binning 1 - { - my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE " . RECIPE1; # Command to run - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppImage: $error_code", $exp_tag, $error_code); - } - &my_die("Unable to find expected output file: $jpeg1", $exp_tag, $PS_EXIT_PROG_ERROR) unless -f $jpeg1; - } - - # Make the jpeg for binning 2 - { - my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE " . RECIPE2; # Command to run - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppImage: $error_code", $exp_tag, $error_code); - } - &my_die("Unable to find expected output file: $jpeg2", $exp_tag, $PS_EXIT_PROG_ERROR) unless -f $jpeg2; - } -} - -# XXX keep the same outroot as the input -# Add the result into the database -$outputRoot = $ipprc->convert_filename_relative($outputRoot); -$jpeg1 = $ipprc->convert_filename_relative($jpeg1); -$jpeg2 = $ipprc->convert_filename_relative($jpeg2); - -unless ($no_update) { - my $command = "$p3tool -addprocessedexp -exp_tag $exp_tag -uri UNKNOWN " . - "-recip " . RECIPE1() . "," . RECIPE2() . " -b1_uri $outputRoot " . - "-bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev " . - "-sigma_ra 0.0 -sigma_dec 0.0 -nastro 0 -zp_mean 0.0 -zp_stdev 0.0"; # Command to run - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to perform dettool -addprocessedexp: $error_code\n"); - exit($error_code); - } - - unlink $list1Name; - unlink $list2Name; -} - - -sub my_die -{ - my $msg = shift; # Warning message on die - my $exp_tag = shift; # Exposure tag - my $exit_code = shift; # Exit code to add - - warn($msg); - if ($exp_tag and not $no_update) { - my $command = "$p3tool -addprocessedexp -exp_tag $exp_tag -code $exit_code"; - $command .= " -dbname $dbname" if defined $dbname; - system ($command); - } - exit $exit_code; -} - - -END { - my $status = $?; - system("sync") == 0 - or die "failed to execute sync: $!" ; - $? = $status; -} - -__END__ Index: unk/ippScripts/scripts/phase4_overlap.pl =================================================================== --- /trunk/ippScripts/scripts/phase4_overlap.pl (revision 12012) +++ (revision ) @@ -1,146 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -## report the program and machine -use Sys::Hostname; -my $host = hostname(); -print "\n\n"; -print "Starting script $0 on $host\n\n"; - -use vars qw( $VERSION ); -$VERSION = '0.01'; - -use IPC::Cmd qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; - -use PS::IPP::Config qw( - $PS_EXIT_SUCCESS - $PS_EXIT_UNKNOWN_ERROR - $PS_EXIT_SYS_ERROR - $PS_EXIT_CONFIG_ERROR - $PS_EXIT_PROG_ERROR - $PS_EXIT_DATA_ERROR - $PS_EXIT_TIMEOUT_ERROR - ); -my $ipprc = PS::IPP::Config->new(); # IPP configuration -use File::Spec; - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -my ($p4_id, $dbname, $workdir, $no_update, $no_op); -GetOptions( - 'p4_id|i=s' => \$p4_id, # Phase 4 identifier - 'dbname|d=s' => \$dbname, # Database name - 'workdir|w=s' => \$workdir, # Working directory, for output files - 'no-update' => \$no_update, # Don't update the database? - 'no-op' => \$no_op, # Don't do any operations -) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( - -msg => "Required options: --p4_id", - -exitval => 3, -) unless defined $p4_id; - -# Look for programs we need -my $missing_tools; -my $p4tool = can_run('p4tool') or (warn "Can't find p4tool" and $missing_tools = 1); -#my $overlap = can_run('overlap') or (warn "Can't find overlap" and $missing_tools = 1); -if ($missing_tools) { - warn("Can't find required tools."); - exit($PS_EXIT_CONFIG_ERROR); -} - -# Get list of component imfiles for exposure -my $imfiles; -{ - my $command = "$p4tool -imfile -p4_id $p4_id"; - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform p4tool -imfile: $error_code", $p4_id, $error_code); - } - - my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $p4_id, $PS_EXIT_PROG_ERROR); - $imfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $p4_id, $PS_EXIT_PROG_ERROR); -} - -unless ($no_op) { - -### Calculates, in some as-yet unknown way the overlaps between imfiles and skycells - -} else { - # Add in the skycell and tesselation for each imfile - foreach my $imfile (@$imfiles) { - $imfile->{skycell_id} = 'default'; - $imfile->{tess_id} = 'default'; - } -} - - -# Generate a MDC file with the overlaps -$workdir = $ipprc->convert_filename_absolute( $workdir ); -my $overlapName = File::Spec->catfile( $workdir, $p4_id . '.overlap.mdc' ); -open my $overlapFile, "> $overlapName" or - &my_die("Unable to open mdc file $overlapName", $p4_id, $PS_EXIT_DATA_ERROR); -print $overlapFile "p4SkyCellMap MULTI\n\n"; -foreach my $imfile (@$imfiles) { - print $overlapFile "p4_id\t\tS32\t" . $imfile->{p4_id} . "\n"; - print $overlapFile "skycell_id\tSTR\t" . $imfile->{skycell_id} . "\n"; - print $overlapFile "tess_id\t\tSTR\t" . $imfile->{tess_id} . "\n"; - print $overlapFile "exp_tag\t\tSTR\t" . $imfile->{exp_tag} . "\n"; - print $overlapFile "p3_version\tS32\t" . $imfile->{p3_version} . "\n"; - print $overlapFile "class_id\t\tSTR\t" . $imfile->{class_id} . "\n"; -} -close $overlapFile; - -# Add the processed file to the database -unless ($no_update) { - my $command = "$p4tool -addoverlap -p4_id $p4_id -info $overlapName"; # Command to run p4tool - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to perform p4tool -addoverlap: $error_code\n"); - exit($error_code); - } - - unlink $overlapName; -} - - - -sub my_die -{ - my $msg = shift; # Warning message on die - my $p4_id = shift; # Phase 4 identifier - my $exit_code = shift; # Exit code to add - - warn($msg); - if ($p4_id and not $no_update) { - my $command = "$p4tool -addoverlap -p4_id $p4_id -code $exit_code"; - $command .= " -dbname $dbname" if defined $dbname; - system ($command); - } - exit $exit_code; -} - -END { - my $status = $?; - system("sync") == 0 - or die "failed to execute sync: $!" ; - $? = $status; -} - -__END__ Index: unk/ippScripts/scripts/phase4_warp.pl =================================================================== --- /trunk/ippScripts/scripts/phase4_warp.pl (revision 12012) +++ (revision ) @@ -1,181 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -## report the program and machine -use Sys::Hostname; -my $host = hostname(); -print "\n\n"; -print "Starting script $0 on $host\n\n"; - -use vars qw( $VERSION ); -$VERSION = '0.01'; - -use IPC::Cmd qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; - -use PS::IPP::Config qw( - $PS_EXIT_SUCCESS - $PS_EXIT_UNKNOWN_ERROR - $PS_EXIT_SYS_ERROR - $PS_EXIT_CONFIG_ERROR - $PS_EXIT_PROG_ERROR - $PS_EXIT_DATA_ERROR - $PS_EXIT_TIMEOUT_ERROR - ); -my $ipprc = PS::IPP::Config->new(); # IPP configuration -use File::Spec; - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -my ($p4_id, $skycell_id, $camera, $dbname, $workdir, $no_update, $no_op); -GetOptions( - 'p4_id|i=s' => \$p4_id, # Phase 4 identifier - 'skycell_id|s=s' => \$skycell_id, # Skycell identifier - 'camera|c=s' => \$camera, # Camera name - 'dbname|d=s' => \$dbname, # Database name - 'workdir|w=s' => \$workdir, # Working directory, for output files - 'no-update' => \$no_update, # Don't update the database? - 'no-op' => \$no_op, # Don't do any operations? -) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( - -msg => "Required options: --p4_id --skycell_id --camera", - -exitval => 3, -) unless defined $p4_id - and defined $skycell_id - and defined $camera; - -$ipprc->define_camera($camera); - -# Look for programs we need -my $missing_tools; -my $p4tool = can_run('p4tool') or (warn "Can't find p4tool" and $missing_tools = 1); -my $pswarp = can_run('pswarp') or (warn "Can't find pswarp" and $missing_tools = 1); -if ($missing_tools) { - warn("Can't find required tools."); - exit($PS_EXIT_CONFIG_ERROR); -} - -# Get list of component imfiles for exposure -my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -my $imfiles; -{ - my $command = "$p4tool -imfile -p4_id $p4_id -skycell_id $skycell_id"; - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform p4tool -imfile: $error_code", $p4_id, $skycell_id, $error_code); - } - - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $p4_id, $skycell_id, $PS_EXIT_PROG_ERROR); - $imfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $p4_id, $skycell_id, $PS_EXIT_PROG_ERROR); -} - -### Output file name -if (defined $workdir) { - $workdir = $ipprc->convert_filename_absolute( $workdir ); -} else { - my ($vol, $dir, $file) = File::Spec->splitpath( $$imfiles[0]->{uri} ); - $workdir = $dir; -} -my $outputFile = "$skycell_id.warp.$p4_id"; # Root name -my $outputRoot = File::Spec->catfile( $workdir, $outputFile ); -my $outputImage = $ipprc->filename("PSWARP.OUTPUT", $outputRoot, $skycell_id ); -my $outputBin1 = $ipprc->filename("PSWARP.BIN1", $outputRoot, $skycell_id ); -my $outputBin2 = $ipprc->filename("PSWARP.BIN2", $outputRoot, $skycell_id ); -my $outputStats = $outputRoot . '.stats'; -my $listName = $outputRoot . '.list'; - -# Get list of filenames -open my $listFile, '>' . $listName; -foreach my $imfile (@$imfiles) { - my $uri = $ipprc->convert_filename_absolute( $imfile->{uri} ); - - print $listFile "$uri\n"; -} -close $listFile; - -# Run pswarp -my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser -unless ($no_op) { - my $command = "$pswarp -list $listName $outputRoot -stat $outputStats"; # Command to run pswarp - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform pswarp: $error_code", $p4_id, $skycell_id, $error_code); - } - &my_die("Couldn't find expected output file: $outputImage", $p4_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputImage; - &my_die("Couldn't find expected output file: $outputStats", $p4_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputStats; - &my_die("Couldn't find expected output file: $outputBin1", $p4_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin1; - &my_die("Couldn't find expected output file: $outputBin2", $p4_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin2; - - # Get the statistics on the warped image - my $statsFile; # File handle - open $statsFile, "$outputStats" or die "Can't open statistics file $outputStats: $!\n"; - my @contents = <$statsFile>; # Contents of file - close $statsFile; - my $metadata = $mdcParser->parse(join "", @contents) - or &my_die("Unable to parse metadata config", $p4_id, $skycell_id, $PS_EXIT_PROG_ERROR); - $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $p4_id, $skycell_id, $PS_EXIT_PROG_ERROR); -} - -# Take off the absolute path, to stuff into the database -$outputImage = $ipprc->convert_filename_relative( $outputImage ); -$outputBin1 = $ipprc->convert_filename_relative( $outputBin1 ); -$outputBin2 = $ipprc->convert_filename_relative( $outputBin2 ); - -my $bg = ($stats->bg_mean() or 'NAN'); -my $bg_stdev = ($stats->bg_stdev() or 'NAN'); - -# Add the processed file to the database -unless ($no_update) { - my $command = "$p4tool -warped -p4_id $p4_id -skycell_id $skycell_id" . - " -uri $outputImage -b1_uri $outputRoot"; # Command to run dettool - $command .= " -bg $bg -bg_stdev $bg_stdev"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to perform p4tool -addwarpedimfile: $error_code\n"); - exit($error_code); - } - - unlink $outputStats; -} - -sub my_die -{ - my $msg = shift; # Warning message on die - my $p4_id = shift; # Phase 4 identifier - my $skycell_id = shift; # Skycell identifier - my $exit_code = shift; # Exit code to add - - warn($msg); - if ($p4_id and $skycell_id and not $no_update) { - my $command = "$p4tool -warped -p4_id $p4_id -skycell_id $skycell_id -code $exit_code"; - $command .= " -dbname $dbname" if defined $dbname; - system ($command); - } - exit $exit_code; -} - -END { - my $status = $?; - system("sync") == 0 - or die "failed to execute sync: $!" ; - $? = $status; -} - -__END__ Index: unk/ippScripts/scripts/phase5_subtract.pl =================================================================== --- /trunk/ippScripts/scripts/phase5_subtract.pl (revision 12012) +++ (revision ) @@ -1,213 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -## report the program and machine -use Sys::Hostname; -my $host = hostname(); -print "\n\n"; -print "Starting script $0 on $host\n\n"; - -use vars qw( $VERSION ); -$VERSION = '0.01'; - -use IPC::Cmd qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; -use Data::Dumper; - -use PS::IPP::Config qw( - $PS_EXIT_SUCCESS - $PS_EXIT_UNKNOWN_ERROR - $PS_EXIT_SYS_ERROR - $PS_EXIT_CONFIG_ERROR - $PS_EXIT_PROG_ERROR - $PS_EXIT_DATA_ERROR - $PS_EXIT_TIMEOUT_ERROR - ); -my $ipprc = PS::IPP::Config->new(); # IPP configuration -use File::Spec; - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -my ($p5_id, $camera, $dbname, $workdir, $no_update, $no_op); -GetOptions( - 'p5_id|d=s' => \$p5_id, # Phase 5 identifier - 'camera|c=s' => \$camera, # Camera name - 'dbname|d=s' => \$dbname, # Database name - 'workdir|w=s' => \$workdir, # Working directory, for output files - 'no-update' => \$no_update, # Don't update the database? - 'no-op' => \$no_op, # Don't do any operations? -) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( - -msg => "Required options: --p5_id --camera", - -exitval => 3, -) unless defined $p5_id - and defined $camera; - -$ipprc->define_camera($camera); - -# Look for programs we need -my $missing_tools; -my $p5tool = can_run('p5tool') or (warn "Can't find p5tool" and $missing_tools = 1); -my $pois = can_run('pois') or (warn "Can't find pois" and $missing_tools = 1); -if ($missing_tools) { - warn("Can't find required tools."); - exit($PS_EXIT_CONFIG_ERROR); -} - -# Get list of components for subtraction -my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -my $files; -{ - my $command = "$p5tool -inputscfile -p5_id $p5_id"; - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform p5tool -inputscfile: $error_code", $p5_id, $error_code); - } - - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $p5_id, $PS_EXIT_PROG_ERROR); - $files = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $p5_id, $PS_EXIT_PROG_ERROR); -} - -&my_die("Subtraction list contains more than two elements", $p5_id, $PS_EXIT_SYS_ERROR) unless - scalar @$files == 2; - -# Identify the input and the template -my $input; # Input file -my $template; # Template file -my $skycell_id; # Skycell identifier -foreach my $file (@$files) { - my $uri = $ipprc->convert_filename_absolute( $file->{uri} ); # URI for file - if (defined $file->{template} and $file->{template}) { - $template = $uri; - } else { - $input = $uri; - } - if (defined $skycell_id) { - &my_die("Skycell identifiers don't match", $p5_id, $PS_EXIT_SYS_ERROR) unless - $file->{skycell_id} eq $skycell_id; - } else { - $skycell_id = $file->{skycell_id}; - } -} - -&my_die("Unable to identify template", $p5_id, $PS_EXIT_SYS_ERROR) unless defined $template; -&my_die("Unable to identify input", $p5_id, $PS_EXIT_SYS_ERROR) unless defined $input; - -### Working directory -if (defined $workdir) { - $workdir = $ipprc->convert_filename_absolute( $workdir ); -} else { - my ($vol, $dir, $file) = File::Spec->splitpath( $input ); - $workdir = $dir; -} - -# Get the output filenames -my $outputFile = "$skycell_id.sub.$p5_id"; # Root name -my $outputRoot = File::Spec->catfile( $workdir, $outputFile ); - -my $outputName = $outputRoot . ".fits"; -my $bin1Name = $ipprc->filename("PPIMAGE.BIN1", $outputRoot); -my $bin2Name = $ipprc->filename("PPIMAGE.BIN2", $outputRoot); -my $outputStats = $outputRoot . '.stats'; - -# Perform subtraction -my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser -unless ($no_op) { - my $command = "$pois $template $input $outputName"; # Command to run pois - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppImage: $error_code", $p5_id, $error_code); - } - &my_die("Couldn't find expected output file: $outputName", $p5_id, $PS_EXIT_SYS_ERROR) unless -f $outputName; -# &my_die("Couldn't find expected output file: $bin1Name", $p5_id, $PS_EXIT_SYS_ERROR) unless -f $bin1Name; -# &my_die("Couldn't find expected output file: $bin2Name", $p5_id, $PS_EXIT_SYS_ERROR) unless -f $bin2Name; -# &my_die("Couldn't find expected output file: $outputStats", $p5_id, $PS_EXIT_SYS_ERROR) unless -f $outputStats; - - # Get the statistics on the residual image - if (0) { ### Disabled because pois doesn't output stats yet - my $statsFile; # File handle - open $statsFile, $outputStats or &my_die("Can't open statistics file $outputStats: $!", $p5_id, $PS_EXIT_SYS_ERROR); - my @contents = <$statsFile>; # Contents of file - close $statsFile; - my $metadata = $mdcParser->parse(join "", @contents) or - &my_die("Unable to parse metadata config doc", $p5_id, $PS_EXIT_PROG_ERROR); - $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $p5_id, $PS_EXIT_PROG_ERROR); - } -} - -# Add the processed file to the database -$outputName = $ipprc->convert_filename_relative( $outputName ); - -my $bg = ($stats->bg_mean() or 'NAN'); -my $bg_stdev = ($stats->bg_stdev() or 'NAN'); - -unless ($no_update) { - - # Add the subtraction result - { - my $command = "$p5tool -adddiffscfile -p5_id $p5_id -uri $outputName -b1_uri $outputRoot"; - $command .= " -bg $bg -bg_stdev $bg_stdev"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform p5tool -adddiffscfile: $error_code", $p5_id, $error_code); - } - -# unlink $outputStats; - } - - # Register the run as completed - { - my $command = "$p5tool -updaterun -p5_id $p5_id -state stop"; # Command to run p5tool - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to perform p5tool -updaterun: $error_code\n"); - exit($error_code); - } - } -} - - -sub my_die -{ - my $msg = shift; # Warning message on die - my $p5_id = shift; # Phase 5 identifier - my $exit_code = shift; # Exit code to add - - warn($msg); - if ($p5_id and not $no_update) { - my $command = "$p5tool -updaterun -p5_id $p5_id -state stop -code $exit_code"; - $command .= " -dbname $dbname" if defined $dbname; - system ($command); - } - exit $exit_code; -} - -END { - my $exit = $?; - system("sync") == 0 or die "failed to execute sync: $!"; - $? = $exit; -} - -__END__ Index: unk/ippScripts/scripts/phase6_sum.pl =================================================================== --- /trunk/ippScripts/scripts/phase6_sum.pl (revision 12012) +++ (revision ) @@ -1,205 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -## report the program and machine -use Sys::Hostname; -my $host = hostname(); -print "\n\n"; -print "Starting script $0 on $host\n\n"; - -use vars qw( $VERSION ); -$VERSION = '0.01'; - -use IPC::Cmd qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; -use Data::Dumper; - -use PS::IPP::Config qw( - $PS_EXIT_SUCCESS - $PS_EXIT_UNKNOWN_ERROR - $PS_EXIT_SYS_ERROR - $PS_EXIT_CONFIG_ERROR - $PS_EXIT_PROG_ERROR - $PS_EXIT_DATA_ERROR - $PS_EXIT_TIMEOUT_ERROR - ); -my $ipprc = PS::IPP::Config->new(); # IPP configuration -use File::Spec; - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -my ($p6_id, $camera, $dbname, $workdir, $no_update, $no_op); -GetOptions( - 'p6_id|d=s' => \$p6_id, # Phase 6 identifier - 'camera|c=s' => \$camera, # Camera name - 'dbname|d=s' => \$dbname, # Database name - 'workdir|w=s' => \$workdir, # Working directory, for output files - 'no-update' => \$no_update, # Don't update the database? - 'no-op' => \$no_op, # Don't do any operations? -) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( - -msg => "Required options: --p6_id --camera", - -exitval => 3, -) unless defined $p6_id - and defined $camera; - -$ipprc->define_camera($camera); - -# Look for programs we need -my $missing_tools; -my $p6tool = can_run('p6tool') or (warn "Can't find p6tool" and $missing_tools = 1); -my $ppStac = can_run('ppStac') or (warn "Can't find ppStac" and $missing_tools = 1); -if ($missing_tools) { - warn("Can't find required tools."); - exit($PS_EXIT_CONFIG_ERROR); -} - -# Get list of components for subtraction -my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -my $files; -{ - my $command = "$p6tool -inputscfile -p6_id $p6_id"; - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform p6tool -inputscfile: $error_code", $p6_id, $error_code); - } - - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $p6_id, $PS_EXIT_PROG_ERROR); - $files = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $p6_id, $PS_EXIT_PROG_ERROR); -} - -&my_die("Subtraction list contains more than two elements", $p6_id, $PS_EXIT_SYS_ERROR) unless - scalar @$files >= 2; - -# Identify the input and the template -my $inputList; # List of input files -my $skycell_id; # Skycell identifier -foreach my $file (@$files) { - my $uri = $ipprc->convert_filename_absolute( $file->{uri} ); # URI for file - $inputList .= "$uri "; - if (defined $skycell_id) { - &my_die("Skycell identifiers don't match", $p6_id, $PS_EXIT_SYS_ERROR) unless - $file->{skycell_id} eq $skycell_id; - } else { - $skycell_id = $file->{skycell_id}; - } -} - -### Working directory -if (defined $workdir) { - $workdir = $ipprc->convert_filename_absolute( $workdir ); -} else { - my ($vol, $dir, $file) = File::Spec->splitpath( $ipprc->convert_filename_absolute( $$files[0]->{uri} ) ); - $workdir = $dir; -} - -# Get the output filenames -my $outputFile = "$skycell_id.sub.$p6_id"; # Root name -my $outputRoot = File::Spec->catfile( $workdir, $outputFile ); - -my $outputName = $outputRoot . ".fits"; -my $bin1Name = $ipprc->filename("PPIMAGE.BIN1", $outputRoot); -my $bin2Name = $ipprc->filename("PPIMAGE.BIN2", $outputRoot); -my $outputStats = $outputRoot . '.stats'; - -# Perform subtraction -my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser -unless ($no_op) { - my $command = "$ppStac $outputName $inputList"; # Command to run pois - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppImage: $error_code", $p6_id, $error_code); - } - &my_die("Couldn't find expected output file: $outputName", $p6_id, $PS_EXIT_SYS_ERROR) unless -f $outputName; -# &my_die("Couldn't find expected output file: $bin1Name", $p6_id, $PS_EXIT_SYS_ERROR) unless -f $bin1Name; -# &my_die("Couldn't find expected output file: $bin2Name", $p6_id, $PS_EXIT_SYS_ERROR) unless -f $bin2Name; -# &my_die("Couldn't find expected output file: $outputStats", $p6_id, $PS_EXIT_SYS_ERROR) unless -f $outputStats; - - # Get the statistics on the residual image - if (0) { ### Disabled because ppStac doesn't output stats yet - my $statsFile; # File handle - open $statsFile, $outputStats or &my_die("Can't open statistics file $outputStats: $!", $p6_id, $PS_EXIT_SYS_ERROR); - my @contents = <$statsFile>; # Contents of file - close $statsFile; - my $metadata = $mdcParser->parse(join "", @contents) or - &my_die("Unable to parse metadata config doc", $p6_id, $PS_EXIT_PROG_ERROR); - $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $p6_id, $PS_EXIT_PROG_ERROR); - } -} - -# Add the processed file to the database -$outputName = $ipprc->convert_filename_relative( $outputName ); - -my $bg = ($stats->bg_mean() or 'NAN'); -my $bg_stdev = ($stats->bg_stdev() or 'NAN'); - -unless ($no_update) { - - # Add the subtraction result - { - my $command = "$p6tool -addsumscfile -p6_id $p6_id -uri $outputName -b1_uri $outputRoot"; - $command .= " -bg $bg -bg_stdev $bg_stdev"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform p6tool -adddiffscfile: $error_code", $p6_id, $error_code); - } - -# unlink $outputStats; - } - - # Register the run as completed - { - my $command = "$p6tool -updaterun -p6_id $p6_id -state stop"; # Command to run p6tool - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 1); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to perform p6tool -updaterun: $error_code\n"); - exit($error_code); - } - } -} - - -sub my_die -{ - my $msg = shift; # Warning message on die - my $p6_id = shift; # Phase 5 identifier - my $exit_code = shift; # Exit code to add - - warn($msg); - if ($p6_id and not $no_update) { - my $command = "$p6tool -updaterun -p6_id $p6_id -state stop -code $exit_code"; - $command .= " -dbname $dbname" if defined $dbname; - system ($command); - } - exit $exit_code; -} - -END { - my $exit = $?; - system("sync") == 0 or die "failed to execute sync: $!"; - $? = $exit; -} - -__END__ Index: /trunk/ippScripts/scripts/register_exp.pl =================================================================== --- /trunk/ippScripts/scripts/register_exp.pl (revision 12013) +++ /trunk/ippScripts/scripts/register_exp.pl (revision 12013) @@ -0,0 +1,267 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use Cache::File; +use Storable qw( freeze thaw ); +use File::Basename qw( basename ); +use IPC::Cmd qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Statistics::Descriptive; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my ($cache, $exptag, $workdir, $dbname, $no_update); + +GetOptions( + 'caches' => \$cache, + 'exp_tag|e=s' => \$exptag, + 'workdir|w=s' => \$workdir, # Working directory for output files + 'dbname|d=s' => \$dbname, # Database name + 'no-update' => \$no_update +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --exp_tag", + -exitval => 3, +) unless defined $exptag; + +# Define setup +use constant TYPE => "exp_type"; # Observation type keyword +use constant DETRENDS => [ "bias", "zero", "dark", "shutter", "flat", "fringe", "domeflat", "skyflat" ]; # Observation types to mark as detrend +use constant DETREND_FLAG => "-detrend"; # Flag to use to mark detrend exposure + +use constant PHASE0_URI => 'uri'; # Key for the URI +use constant PHASE0_CLASSID => 'class_id'; # Key for the class id +use constant PHASE0_BG => 'bg'; # Key for the background +use constant PHASE0_BG_STDEV => 'bg_stdev'; # Key for the background standard deviation +use constant PHASE0_BG_MEAN_STDEV => 'bg_mean_stdev'; # Key for the mean of the background standard deviation + +# These values should be constant for all components +use constant CONSTANTS => [ + "object", # Object + "exp_type", # Exposure type + "filter", # Filter used + "airmass", # Airmass + "ra", # Right ascension + "decl", # Declination + "posang", # Position angle + "alt", # Altitude + "az", # Azimuth + "ccd_temp", # CCD temperature + "dateobs", # Time of exposure + ]; + +# These values may vary across components; we will take the average +use constant VARIABLES => [ + "exp_time", # Exposure time + ]; + + +# Look for commands we need +my $missing_tools; +my $regtool = can_run('p0tool') + or (warn "can't find regtool" and $missing_tools = 1); +my $ppStats = can_run('ppStats') + or (warn "can't find ppStats" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# setup cache interface + +my $c = Cache::File->new( + cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)), + default_expires => '7200 sec', +); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get the list of imfiles +my $imfiles; +{ + my $command = "$regtool -processedimfile -exp_tag $exptag"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool on exposure id $exptag: $error_code"); + exit ($error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + unless ($metadata) { + warn ("Unable to parse metadata config doc"); + &my_die ($exptag, $PS_EXIT_PROG_ERROR); + } + $imfiles = parse_md_list($metadata); # Data for imfiles +} + +# Process the values +my %values; # Values to update +foreach my $variable (@{VARIABLES()}) { + $values{$variable} = []; +} +my @backgrounds; # Array of backgrounds +my @variances; # Array of background standard deviations\ +my $obsType; # Observation type + +foreach my $imfile (@$imfiles) { + foreach my $constant (@{CONSTANTS()}) { + my $value = get_value($imfile, $constant); # Value for imfile + if (not defined $values{$constant}) { + $values{$constant} = $value; + } + if ($values{$constant} ne $value) { + warn ("Value of $constant for $imfile->{PHASE0_CLASSID} doesn't match previous value"); + &my_die ($exptag, $PS_EXIT_PROG_ERROR); + } + } + + foreach my $variable (@{VARIABLES()}) { + my $value = get_value($imfile, $variable); # Value for imfile + my $array = $values{$variable}; # Array of data + push @$array, $value; + } + + my $bg = get_value($imfile, PHASE0_BG()); + push @backgrounds, $bg; + + my $stdev = get_value($imfile, PHASE0_BG_MEAN_STDEV()); + push @variances, $stdev**2; +} + +if (0) { +# XXX for a test, randomly declare a failure and return to pantasks +my $rnd = rand(1); +if ($rnd > 0.5) { + warn ("random failure"); + &my_die ($exptag, $PS_EXIT_DATA_ERROR); +} +} + +# Output results to the database +unless ($no_update) { + my @command; + push @command, $regtool, '-addprocessedexp', '-exp_tag', $exptag; # Command to execute to update exposure parameters + + # Add the values of interest + foreach my $constant (@{CONSTANTS()}) { + push @command, ( '-' . $constant ), $values{$constant}; + } + foreach my $variable (@{VARIABLES()}) { + my $array = $values{$variable}; # Array of values + my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator + $stats->add_data(@$array); + push @command, ( '-' . $variable ), $stats->mean(); + } + + # Add the statistics + { + my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator + $stats->add_data(@backgrounds); + push @command, ( '-' . PHASE0_BG() ), $stats->mean(); + push @command, ( '-' . PHASE0_BG_MEAN_STDEV() ), ( $stats->standard_deviation() || 0 ) + } + { + my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator + $stats->add_data(@variances); + push @command, ( '-' . PHASE0_BG_STDEV() ), sqrt( $stats->mean() ); + } + + # Add the detrend flag + foreach my $detrendType (@{DETRENDS()}) { + if (lc($values{TYPE()}) =~ /$detrendType/) { + push @command, DETREND_FLAG; + last; + } + } + + # Quote arguments with whitespace + for (my $i = 0; $i < scalar @command; $i++) { + if ($command[$i] =~ /\s/) { + $command[$i] = "\'$command[$i]\'"; + } + } + + push @command, "-dbname", $dbname if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => \@command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to run phase0 update for $exptag: $error_code"); + exit($error_code); + } +} + +### Pau. + +# Get the value for a particular imfile, along with a strong check for its existence +sub get_value { + my $imfile = shift; # The hash of values + my $name = shift; # The name of the value to check + + my $source = $imfile->{PHASE0_CLASSID()}; # Where it comes from + + unless (defined $imfile->{$name}) { + warn ("Couldn't find value of $name for class_id=$source"); + &my_die ($exptag, $PS_EXIT_PROG_ERROR); + } + return $imfile->{$name}; +} + +sub cache_run +{ + my %p = @_; + + my $cmd_output = $c->get($p{command}) if $cache; + if (defined $cmd_output) { + return @{thaw $cmd_output}; + } else { + my @output = run(%p); + $c->set($p{command}, freeze \@output) if $cache; + return @output; + } +} + +sub my_die +{ + my $exp_tag = $_[0]; + my $exit_code = $_[1]; + if ($exp_tag and not $no_update) { + my $command = "$regtool -addprocessedexp -exp_tag $exp_tag -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} Index: /trunk/ippScripts/scripts/register_imfile.pl =================================================================== --- /trunk/ippScripts/scripts/register_imfile.pl (revision 12013) +++ /trunk/ippScripts/scripts/register_imfile.pl (revision 12013) @@ -0,0 +1,250 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Cache::File; +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use Data::Dumper; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($cache, $exp_tag, $class_id, $uri, $workdir, $dbname, $no_update); + +# print "args: @ARGV\n"; +# for (my $i = 0; $i < @ARGV; $i++) { +# print "arg $i: ...$ARGV[$i]...\n"; +# } + +GetOptions( + 'caches' => \$cache, + 'exp_tag|e=s' => \$exp_tag, + 'class_id|i=s' => \$class_id, + 'uri|u=s' => \$uri, + 'workdir|w=s' => \$workdir, # Working directory for output files + 'dbname|d=s' => \$dbname,# Database name + 'no-update' => \$no_update +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --exp_tag --class_id --uri", + -exitval => 3, +) unless defined $exp_tag + and defined $class_id + and defined $uri; + +my $RECIPE = "PPSTATS_PHASE0"; # Recipe to use for ppStats +# use constant RECIPE => "PPSTATS_PHASE0"; # Recipe to use for ppStats + +# These values should be constant for all components +# The key is the name in the ppStats output; the value is the switch for "phase0 -updateexp" +use constant CONSTANTS => { + "FPA.OBJECT" => "-object", # Object + "FPA.OBSTYPE" => "-exp_type", # Exposure type + "FPA.FILTER" => "-filter", # Filter used + "FPA.AIRMASS" => "-airmass", # Airmass + "FPA.RA" => "-ra", # Right ascension + "FPA.DEC" => "-decl", # Declination + "FPA.ALT" => "-alt", # Altitude + "FPA.AZ" => "-az", # Azimuth + "FPA.POSANGLE" => "-posang", # Position angle + "FPA.TIME" => "-dateobs", # Date of observation (UTC) + }; + +# These values may vary across components; we will take the average +# The key is the name in the ppStats output; the value is the switch for "phase0 -updateexp" +use constant VARIABLES => { + "CHIP.TEMP" => "-ccd_temp", # CCD temperature + "CELL.EXPOSURE" => "-exp_time" # Exposure time + }; + + +# Switches for regtool +use constant P0TOOL_MODE => '-addprocessedimfile'; # Mode for regtool +use constant P0TOOL_EXPTAG => '-exp_tag'; # Switch to specify the exposure id +use constant P0TOOL_CLASSID => '-class_id'; # Switch to specify the class id +use constant P0TOOL_BG_MEAN => '-bg'; # Switch to add the background +use constant P0TOOL_BG_STDEV => '-bg_stdev'; # Switch to add the bg stdev +use constant P0TOOL_BG_MEAN_STDEV => '-bg_mean_stdev'; # Switch to add the bg mean stdev + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('p0tool') + or (warn "Can't find regtool" and $missing_tools = 1); +my $ppStats = can_run('ppStats') + or (warn "Can't find ppStats" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# setup cache interface + +my $c = Cache::File->new( + cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)), + default_expires => '7200 sec', +); + +# Resolve the input URI +$uri = $ipprc->convert_filename_absolute($uri); + +# Run ppStats on the input file +my $stats; +{ + my $command = "$ppStats $uri -recipe PPSTATS $RECIPE"; # Command to run ppStats + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform ppStats on exposure id $exp_tag: $error_code"); + &my_die ($exp_tag, $class_id, $error_code); + } + + # Parse the output + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf); # XXX is this join necessary? + unless ($metadata) { + warn ("Unable to parse metadata config doc"); + &my_die ($exp_tag, $class_id, $PS_EXIT_PROG_ERROR); + } + my @constants = keys %{CONSTANTS()}; # List of constants to parse out + my @variables = keys %{VARIABLES()}; # List of variables to parse out + $stats = PS::IPP::Metadata::Stats->new(\@constants, \@variables); # Stats parser + unless ($stats->parse($metadata)) { + warn ("Unable to find all values"); + &my_die ($exp_tag, $class_id, $PS_EXIT_PROG_ERROR); + # XXX is this a programming or a config error? + } + + if (0) { + # XXX for a test, randomly declare a failure and return to pantasks + my $rnd = rand(1); + if ($rnd > 0.5) { + warn ("random failure"); + &my_die ($exp_tag, $class_id, $PS_EXIT_DATA_ERROR); + } +} +} + +# Push the results into the database +unless ($no_update) { + my @command; + push @command, $regtool, P0TOOL_MODE(), P0TOOL_EXPTAG(), $exp_tag, P0TOOL_CLASSID(), $class_id; # Command to run p0tool + + foreach my $constant (keys %{CONSTANTS()}) { + push @command, CONSTANTS->{$constant}, ($stats->data($constant))->{value}; + + } + foreach my $variable (keys %{VARIABLES()}) { + # Just use the mean value + push @command, VARIABLES->{$variable}, ($stats->data($variable))->{mean}; + } + + push @command, P0TOOL_BG_MEAN(); + if (defined $stats->bg_mean()) { + push @command, $stats->bg_mean(); + } else { + push @command, "NAN"; + } + push @command, P0TOOL_BG_STDEV(); + if (defined($stats->bg_stdev())) { + push @command, $stats->bg_stdev(); + } else { + # Will not be defined if there is only a single imfile + push @command, 0; + } + push @command, P0TOOL_BG_MEAN_STDEV(); + if (defined $stats->bg_mean_stdev()) { + push @command, $stats->bg_mean_stdev(); + } else { + push @command, "0.0"; + } + + # Quote arguments with whitespace + for (my $i = 0; $i < scalar @command; $i++) { + if ($command[$i] =~ /\s/) { + $command[$i] = "\'$command[$i]\'"; + } + } + + push @command, "-dbname", $dbname if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => \@command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + # XXX if we can't run -addprocessedimfile, we can't actually set the + # error code. + # XXX if this is not a database error, it is probably a + # programming error (in regtool or the passed args) + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} + +sub cache_run +{ + my %p = @_; + + my $cmd_output = $c->get($p{command}) if $cache; + if (defined $cmd_output) { + return @{thaw $cmd_output}; + } else { + my @output = run(%p); + $c->set($p{command}, freeze \@output) if $cache; + return @output; + } +} + +sub my_die +{ + my $exp_tag = $_[0]; + my $class_id = $_[1]; + my $exit_code = $_[2]; + if ($exp_tag && $class_id and not $no_update) { + my $command = "$regtool -addprocessedimfile -exp_tag $exp_tag -class_id $class_id -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /trunk/ippScripts/scripts/stack_skycell.pl =================================================================== --- /trunk/ippScripts/scripts/stack_skycell.pl (revision 12013) +++ /trunk/ippScripts/scripts/stack_skycell.pl (revision 12013) @@ -0,0 +1,205 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use Data::Dumper; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($p6_id, $camera, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'p6_id|d=s' => \$p6_id, # Phase 6 identifier + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --p6_id --camera", + -exitval => 3, +) unless defined $p6_id + and defined $camera; + +$ipprc->define_camera($camera); + +# Look for programs we need +my $missing_tools; +my $stacktool = can_run('p6tool') or (warn "Can't find p6tool" and $missing_tools = 1); +my $ppStac = can_run('ppStac') or (warn "Can't find ppStac" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of components for subtraction +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$stacktool -inputscfile -p6_id $p6_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -inputscfile: $error_code", $p6_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $p6_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $p6_id, $PS_EXIT_PROG_ERROR); +} + +&my_die("Subtraction list contains more than two elements", $p6_id, $PS_EXIT_SYS_ERROR) unless + scalar @$files >= 2; + +# Identify the input and the template +my $inputList; # List of input files +my $skycell_id; # Skycell identifier +foreach my $file (@$files) { + my $uri = $ipprc->convert_filename_absolute( $file->{uri} ); # URI for file + $inputList .= "$uri "; + if (defined $skycell_id) { + &my_die("Skycell identifiers don't match", $p6_id, $PS_EXIT_SYS_ERROR) unless + $file->{skycell_id} eq $skycell_id; + } else { + $skycell_id = $file->{skycell_id}; + } +} + +### Working directory +if (defined $workdir) { + $workdir = $ipprc->convert_filename_absolute( $workdir ); +} else { + my ($vol, $dir, $file) = File::Spec->splitpath( $ipprc->convert_filename_absolute( $$files[0]->{uri} ) ); + $workdir = $dir; +} + +# Get the output filenames +my $outputFile = "$skycell_id.sub.$p6_id"; # Root name +my $outputRoot = File::Spec->catfile( $workdir, $outputFile ); + +my $outputName = $outputRoot . ".fits"; +my $bin1Name = $ipprc->filename("PPIMAGE.BIN1", $outputRoot); +my $bin2Name = $ipprc->filename("PPIMAGE.BIN2", $outputRoot); +my $outputStats = $outputRoot . '.stats'; + +# Perform subtraction +my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser +unless ($no_op) { + my $command = "$ppStac $outputName $inputList"; # Command to run pois + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $p6_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $p6_id, $PS_EXIT_SYS_ERROR) unless -f $outputName; +# &my_die("Couldn't find expected output file: $bin1Name", $p6_id, $PS_EXIT_SYS_ERROR) unless -f $bin1Name; +# &my_die("Couldn't find expected output file: $bin2Name", $p6_id, $PS_EXIT_SYS_ERROR) unless -f $bin2Name; +# &my_die("Couldn't find expected output file: $outputStats", $p6_id, $PS_EXIT_SYS_ERROR) unless -f $outputStats; + + # Get the statistics on the residual image + if (0) { ### Disabled because ppStac doesn't output stats yet + my $statsFile; # File handle + open $statsFile, $outputStats or &my_die("Can't open statistics file $outputStats: $!", $p6_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $metadata = $mdcParser->parse(join "", @contents) or + &my_die("Unable to parse metadata config doc", $p6_id, $PS_EXIT_PROG_ERROR); + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $p6_id, $PS_EXIT_PROG_ERROR); + } +} + +# Add the processed file to the database +$outputName = $ipprc->convert_filename_relative( $outputName ); + +my $bg = ($stats->bg_mean() or 'NAN'); +my $bg_stdev = ($stats->bg_stdev() or 'NAN'); + +unless ($no_update) { + + # Add the subtraction result + { + my $command = "$stacktool -addsumscfile -p6_id $p6_id -uri $outputName -b1_uri $outputRoot"; + $command .= " -bg $bg -bg_stdev $bg_stdev"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -adddiffscfile: $error_code", $p6_id, $error_code); + } + +# unlink $outputStats; + } + + # Register the run as completed + { + my $command = "$stacktool -updaterun -p6_id $p6_id -state stop"; # Command to run p6tool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform stacktool -updaterun: $error_code\n"); + exit($error_code); + } + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $p6_id = shift; # Phase 5 identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if ($p6_id and not $no_update) { + my $command = "$stacktool -updaterun -p6_id $p6_id -state stop -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /trunk/ippScripts/scripts/warp_imfile.pl =================================================================== --- /trunk/ippScripts/scripts/warp_imfile.pl (revision 12013) +++ /trunk/ippScripts/scripts/warp_imfile.pl (revision 12013) @@ -0,0 +1,181 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($p4_id, $skycell_id, $camera, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'p4_id|i=s' => \$p4_id, # Phase 4 identifier + 'skycell_id|s=s' => \$skycell_id, # Skycell identifier + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --p4_id --skycell_id --camera", + -exitval => 3, +) unless defined $p4_id + and defined $skycell_id + and defined $camera; + +$ipprc->define_camera($camera); + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('p4tool') or (warn "Can't find p4tool" and $missing_tools = 1); +my $pswarp = can_run('pswarp') or (warn "Can't find pswarp" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component imfiles for exposure +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $imfiles; +{ + my $command = "$warptool -imfile -p4_id $p4_id -skycell_id $skycell_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -imfile: $error_code", $p4_id, $skycell_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $p4_id, $skycell_id, $PS_EXIT_PROG_ERROR); + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $p4_id, $skycell_id, $PS_EXIT_PROG_ERROR); +} + +### Output file name +if (defined $workdir) { + $workdir = $ipprc->convert_filename_absolute( $workdir ); +} else { + my ($vol, $dir, $file) = File::Spec->splitpath( $$imfiles[0]->{uri} ); + $workdir = $dir; +} +my $outputFile = "$skycell_id.warp.$p4_id"; # Root name +my $outputRoot = File::Spec->catfile( $workdir, $outputFile ); +my $outputImage = $ipprc->filename("PSWARP.OUTPUT", $outputRoot, $skycell_id ); +my $outputBin1 = $ipprc->filename("PSWARP.BIN1", $outputRoot, $skycell_id ); +my $outputBin2 = $ipprc->filename("PSWARP.BIN2", $outputRoot, $skycell_id ); +my $outputStats = $outputRoot . '.stats'; +my $listName = $outputRoot . '.list'; + +# Get list of filenames +open my $listFile, '>' . $listName; +foreach my $imfile (@$imfiles) { + my $uri = $ipprc->convert_filename_absolute( $imfile->{uri} ); + + print $listFile "$uri\n"; +} +close $listFile; + +# Run pswarp +my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser +unless ($no_op) { + my $command = "$pswarp -list $listName $outputRoot -stat $outputStats"; # Command to run pswarp + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform pswarp: $error_code", $p4_id, $skycell_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputImage", $p4_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputImage; + &my_die("Couldn't find expected output file: $outputStats", $p4_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputStats; + &my_die("Couldn't find expected output file: $outputBin1", $p4_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin1; + &my_die("Couldn't find expected output file: $outputBin2", $p4_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin2; + + # Get the statistics on the warped image + my $statsFile; # File handle + open $statsFile, "$outputStats" or die "Can't open statistics file $outputStats: $!\n"; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $metadata = $mdcParser->parse(join "", @contents) + or &my_die("Unable to parse metadata config", $p4_id, $skycell_id, $PS_EXIT_PROG_ERROR); + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $p4_id, $skycell_id, $PS_EXIT_PROG_ERROR); +} + +# Take off the absolute path, to stuff into the database +$outputImage = $ipprc->convert_filename_relative( $outputImage ); +$outputBin1 = $ipprc->convert_filename_relative( $outputBin1 ); +$outputBin2 = $ipprc->convert_filename_relative( $outputBin2 ); + +my $bg = ($stats->bg_mean() or 'NAN'); +my $bg_stdev = ($stats->bg_stdev() or 'NAN'); + +# Add the processed file to the database +unless ($no_update) { + my $command = "$warptool -warped -p4_id $p4_id -skycell_id $skycell_id" . + " -uri $outputImage -b1_uri $outputRoot"; # Command to run dettool + $command .= " -bg $bg -bg_stdev $bg_stdev"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addwarpedimfile: $error_code\n"); + exit($error_code); + } + + unlink $outputStats; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $p4_id = shift; # Phase 4 identifier + my $skycell_id = shift; # Skycell identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if ($p4_id and $skycell_id and not $no_update) { + my $command = "$warptool -warped -p4_id $p4_id -skycell_id $skycell_id -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /trunk/ippScripts/scripts/warp_overlap.pl =================================================================== --- /trunk/ippScripts/scripts/warp_overlap.pl (revision 12013) +++ /trunk/ippScripts/scripts/warp_overlap.pl (revision 12013) @@ -0,0 +1,146 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($p4_id, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'p4_id|i=s' => \$p4_id, # Phase 4 identifier + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --p4_id", + -exitval => 3, +) unless defined $p4_id; + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('p4tool') or (warn "Can't find p4tool" and $missing_tools = 1); +#my $overlap = can_run('overlap') or (warn "Can't find overlap" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component imfiles for exposure +my $imfiles; +{ + my $command = "$warptool -imfile -p4_id $p4_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -imfile: $error_code", $p4_id, $error_code); + } + + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $p4_id, $PS_EXIT_PROG_ERROR); + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $p4_id, $PS_EXIT_PROG_ERROR); +} + +unless ($no_op) { + +### Calculates, in some as-yet unknown way the overlaps between imfiles and skycells + +} else { + # Add in the skycell and tesselation for each imfile + foreach my $imfile (@$imfiles) { + $imfile->{skycell_id} = 'default'; + $imfile->{tess_id} = 'default'; + } +} + + +# Generate a MDC file with the overlaps +$workdir = $ipprc->convert_filename_absolute( $workdir ); +my $overlapName = File::Spec->catfile( $workdir, $p4_id . '.overlap.mdc' ); +open my $overlapFile, "> $overlapName" or + &my_die("Unable to open mdc file $overlapName", $p4_id, $PS_EXIT_DATA_ERROR); +print $overlapFile "p4SkyCellMap MULTI\n\n"; +foreach my $imfile (@$imfiles) { + print $overlapFile "p4_id\t\tS32\t" . $imfile->{p4_id} . "\n"; + print $overlapFile "skycell_id\tSTR\t" . $imfile->{skycell_id} . "\n"; + print $overlapFile "tess_id\t\tSTR\t" . $imfile->{tess_id} . "\n"; + print $overlapFile "exp_tag\t\tSTR\t" . $imfile->{exp_tag} . "\n"; + print $overlapFile "p3_version\tS32\t" . $imfile->{p3_version} . "\n"; + print $overlapFile "class_id\t\tSTR\t" . $imfile->{class_id} . "\n"; +} +close $overlapFile; + +# Add the processed file to the database +unless ($no_update) { + my $command = "$warptool -addoverlap -p4_id $p4_id -info $overlapName"; # Command to run p4tool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addoverlap: $error_code\n"); + exit($error_code); + } + + unlink $overlapName; +} + + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $p4_id = shift; # Phase 4 identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if ($p4_id and not $no_update) { + my $command = "$warptool -addoverlap -p4_id $p4_id -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__