Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/calibrate_dvo.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/calibrate_dvo.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/calibrate_dvo.pl (revision 22293) @@ -0,0 +1,205 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($cal_id, $dvodb, $region, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'cal_id|i=s' => \$cal_id, + 'dvodb|c=s' => \$dvodb, + 'region|r=s' => \$region, + 'dbname|d=s' => \$dbname,# Database name + 'workdir|w=s' => \$workdir, # Working directory for output files + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --cal_id --dvodb --region", + -exitval => 3) unless + defined $cal_id and + defined $dvodb and + defined $region; + +# Look for programs we need +my $missing_tools; +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $uniphot = can_run('uniphot') or (warn "Can't find uniphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $caltool = can_run('caltool') or (warn "Can't find caltool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# select the primary filters from DVO query? +my (@filters) = `photcodeList -average`; + +# parse the region (RAs,RAe:DECs,DECe) : item = +/-NNN.NNNN +my @coords = split (":", $region); +my ($RAs, $RAe) = split (",", $coords[0]); +my ($DECs, $DECe) = split (",", $coords[1]); + +# Run addstar -resort +{ + my $command = "$addstar -resort"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RESORT", $status, $dbname); + } +} + +# Run relphot (filter) for each filter +{ + foreach my $filter (@filters) { + my $command = "$relphot $filter"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELPHOT", $status, $dbname); + } + } +} + +# Run uniphot (filter) for each filter +# XXX skip this one? run less frequently? +if (0) { + foreach my $filter (@filters) { + my $command = "$uniphot $filter"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "UNIPHOT", $status, $dbname); + } + } +} + +{ + my $command = "$relastro -objects"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + } +} + +{ + my $command = "$relastro -images"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELASTRO.IMAGES", $status, $dbname); + } +} + +my $command = "$caltool -addrun"; +$command .= " -cal_id $cal_id"; +$command .= " -region $region"; +$command .= " -last_step RELASTRO.IMAGES"; +$command .= " -state 0"; +$command .= " -dbname $dbname" if defined $dbname; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $cal_id = shift; + my $region = shift; + my $last_step = shift; + my $status = shift; + my $dbname = shift; + + carp($msg); + if (defined $cal_id && defined $region && defined $last_step && defined $status and not $no_update) { + my $command = "$caltool -addcalrun"; + $command .= " -cal_id $cal_id"; + $command .= " -region $region"; + $command .= " -last_step $last_step"; + $command .= " -state $status"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/camera_exp.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/camera_exp.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/camera_exp.pl (revision 22293) @@ -0,0 +1,332 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use File::Temp qw( tempfile ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + metadataLookupStr + metadataLookupBool + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $exp_tag, $cam_id, $camera, $outroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update, + $no_op ); +GetOptions( + 'exp_tag=s' => \$exp_tag, # Exposure identifier + 'cam_id=s' => \$cam_id, # Camtool identifier + 'recipe=s' => \$recipe, # Recipe to use + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'outroot|w=s' => \$outroot, # output file base name + 'reduction=s' => \$reduction, # Reduction class + 'dvodb|w=s' => \$dvodb, # output DVO database + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --exp_tag --cam_id --camera", + -exitval => 3, + ) unless + defined $exp_tag and + defined $cam_id and + defined $outroot and + defined $camera; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1'); # Recipe to use +&my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2'); # Recipe to use +&my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# these should be coming from the psastro results for Nimfile > 1, +my $CHIPSTATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bias", type => "mean", flag => "-bias", dtype => "float" }, + { name => "bias_stdev", type => "rms", flag => "-bias_stdev", dtype => "float" }, + { name => "fringe_0", type => "mean", flag => "-fringe_0", dtype => "float" }, + { name => "fringe_1", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "fringe_0", type => "stdev", flag => "-fringe_2", dtype => "float" }, + { name => "sigma_ra", type => "rms", flag => "-sigma_ra", dtype => "float" }, + { name => "sigma_dec", type => "rms", flag => "-sigma_dec", dtype => "float" }, + { name => "ap_resid", type => "mean", flag => "-ap_resid", dtype => "float" }, + { name => "ap_resid_stdev", type => "rms", flag => "-ap_resid_stdev", dtype => "float" }, + { name => "zp_mean", type => "mean", flag => "-zp_mean", dtype => "float" }, + { name => "zp_stdev", type => "rms", flag => "-zp_stdev", dtype => "float" }, + { name => "fwhm_major", type => "mean", flag => "-fwhm_major", dtype => "float" }, + { name => "fwhm_minor", type => "mean", flag => "-fwhm_minor", dtype => "float" }, + { name => "dtime_detrend", type => "sum", flag => "-dtime_detrend", dtype => "float" }, + { name => "dtime_photom", type => "sum", flag => "-dtime_photom", dtype => "float" }, + { name => "dtime_astrom", type => "sum", flag => "-dtime_astrom", dtype => "float" }, + { name => "n_stars", type => "sum", flag => "-n_stars", dtype => "int" }, + { name => "n_extended", type => "sum", flag => "-n_extended", dtype => "int" }, + { name => "n_cr", type => "sum", flag => "-n_cr", dtype => "int" }, + { name => "n_astrom", type => "sum", flag => "-n_astrom", dtype => "int" }, + + +# these are not defined for the database table camProcessedExp + ]; +my $chipStats = PS::IPP::Metadata::Stats->new($CHIPSTATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); + +# test for addstar and psastro: +my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get list of component files +my $files; # Array of component files +{ + my $command = "$camtool -pendingimfile -cam_id $cam_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", $cam_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $cam_id, $PS_EXIT_PROG_ERROR); + + # extract the stats from the metadata + unless ($chipStats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.\n", $cam_id, $PS_EXIT_PROG_ERROR); + } +} + +### not needed to have such an extensive temp file name. +my ($list1File, $list1Name) = tempfile( "$exp_tag.cm.$cam_id.b1.list.XXXX", UNLINK => 1 ); # For binning 1 +my ($list2File, $list2Name) = tempfile( "$exp_tag.cm.$cam_id.b2.list.XXXX", UNLINK => 1 ); # For binning 2 +my ($list3File, $list3Name) = tempfile( "$exp_tag.cm.$cam_id.b3.list.XXXX", UNLINK => 1 ); # For astrometry + +my $chipObjects; +my $chipObjectsExist = 0; +foreach my $file (@$files) { + # use the path_base as OUTPUT root and convert the filenames with ipprc->filename: + my $class_id = $file->{class_id}; + + # If there is only one chip, we use this name for the input to addstar + $chipObjects = $ipprc->filename("PSASTRO.OUTPUT", $file->{path_base}, $class_id); + print $list1File ($ipprc->filename("PPIMAGE.BIN1", $file->{path_base}, $class_id) . "\n"); + print $list2File ($ipprc->filename("PPIMAGE.BIN2", $file->{path_base}, $class_id) . "\n"); + print $list3File ($ipprc->convert_filename_absolute($chipObjects) . "\n"); + + # if any of the output chip astrometry files exist, we can run psastro / addstar below + if ($ipprc->file_exists($chipObjects)) { + $chipObjectsExist = 1; + } +} +close $list1File; +close $list2File; +close $list3File; + +# Output products +$ipprc->outroot_prepare($outroot); + +my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT.MEF", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); + +# convert supplied DVO database name to UNIX filename +my $dvodbReal; +if (defined $dvodb) { + $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO + $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); +} + +unless ($no_op) { + + ## build the output JPEG images first so we get them even if the astrometry fails + + # Make the jpeg for binning 1 + { + my $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($jpeg1); + } + + # Make the jpeg for binning 2 + { + my $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($jpeg2); + } + + # check recipe for PSASTRO.MOSAIC.MODE; run mosaic astrometry if + # requested by the camera's PSASTRO recipe + my $mosaicAstrom; # run mosaic-level astrometry? + { + my $command = "$ppConfigDump -camera $camera -dump-recipe PSASTRO -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR); + $mosaicAstrom = metadataLookupBool($metadata, 'PSASTRO.MOSAIC.MODE'); + } + + # only run psastro / addstar if any of the output chip astrometry files exist (should we test for successful astrometry?) + if ($chipObjectsExist) { + if ($mosaicAstrom) { + # run psastro +mosastro on the set of chips or copy the chipObjects to fpaObjects + # XXX note that this is wrong if imfiles are cells + # XXX add a ppStats call which will collect the astrometry stats + my $command; + $command = "$psastro -list $list3Name $outroot"; + $command .= " +mosastro -chipastro"; + $command .= " -F PSASTRO.OUTPUT PSASTRO.OUTPUT.MEF"; + $command .= " -tracedest $traceDest -log $logDest"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psastro: $error_code", $cam_id, $error_code); + } + # XXX do we want to give an error if astrometry fails here? + &my_die("Unable to find expected output file: $fpaObjects", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects); + } + + # do we supply chipObjects or fpaObjects to addstar? + # run addstar on either the single chip output or the single fpa output + if (defined $dvodbReal) { + # XXX this construct requires the user to have a valid .ptolemyrc + # XXX which in turn points at ippconfig/dvo.site + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + # XXX this needs to be converted to addstar_client... + + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + my $command; + $command = "$addstar -D CAMERA $camdir -update"; + $command .= " -D CATDIR $dvodbReal"; + + if ($mosaicAstrom) { + my $realFile = $ipprc->file_resolve($fpaObjects); + $command .= " $realFile"; + } else { + $command .= " -list $list3Name"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); + } + } + } +} + +my $fpaCommand = "$camtool -addprocessedexp"; +$fpaCommand .= " -cam_id $cam_id"; +$fpaCommand .= " -uri UNKNOWN"; +$fpaCommand .= " -path_base $outroot"; +$fpaCommand .= " -dbname $dbname" if defined $dbname; +$fpaCommand .= $chipStats->cmdflags(); + +# Add the result into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $fpaCommand\n"; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $cam_id = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $cam_id and not $no_update) { + my $command = "$camtool -addprocessedexp -cam_id $cam_id -uri UNKNOWN -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/chip_imfile.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/chip_imfile.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/chip_imfile.pl (revision 22293) @@ -0,0 +1,257 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + metadataLookupStr + metadataLookupBool + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ( $exp_id, $exp_tag, $chip_id, $class_id, $uri, $camera, $outroot, $dbname, $reduction, $verbose, + $no_update, $no_op ); +GetOptions( + 'exp_id=s' => \$exp_id, # Exposure identifier + 'exp_tag=s' => \$exp_tag, # Exposure identifier + 'chip_id=s' => \$chip_id, # Chiptool identifier + 'class_id=s' => \$class_id, # Class identifier + 'uri|u=s' => \$uri, # Input FITS file + 'camera|c=s' => \$camera, # Camera + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --exp_tag --chip_id --class_id --uri --camera --outroot", + -exitval => 3) unless + defined $exp_id and + defined $exp_tag and + defined $chip_id and + defined $class_id and + defined $uri and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, 'CHIP'); # Recipe to use +unless ($recipe) { + &my_die("Couldn't find selected reduction for CHIP: $reduction\n", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +} + +# values to extract from output metadata and the stats to calculate +# XXX commented-out entries are not yet defined in the output files +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "OVER_VAL", type => "mean", flag => "-bias", dtype => "float" }, + { name => "OVER_VAL", type => "stdev", flag => "-bias_stdev", dtype => "float" }, + { name => "FRINGE_0", type => "rms", flag => "-fringe_0", dtype => "float" }, + { name => "FRINGE_RESID_0", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "FRINGE_ERR_0", type => "rms", flag => "-fringe_2", dtype => "float" }, + { name => "CERROR", type => "rms", flag => "-sigma_ra", dtype => "float" }, + { name => "CERROR", type => "rms", flag => "-sigma_dec", dtype => "float" }, + { name => "APMIFIT", type => "mean", flag => "-ap_resid", dtype => "float" }, + { name => "DAPMIFIT", type => "rms", flag => "-ap_resid_stdev", dtype => "float" }, +# { name => "ZP??", type => "mean", flag => "-zp_mean", dtype => "float" }, +# { name => "ZP??", type => "rms", flag => "-zp_stdev", dtype => "float" }, + { name => "FWHM_X", type => "mean", flag => "-fwhm_major", dtype => "float" }, + { name => "FWHM_Y", type => "mean", flag => "-fwhm_minor", dtype => "float" }, + { name => "DT_DET", type => "sum", flag => "-dtime_detrend", dtype => "float" }, + { name => "DT_PHOT", type => "sum", flag => "-dtime_photom", dtype => "float" }, + { name => "DT_ASTR", type => "sum", flag => "-dtime_astrom", dtype => "float" }, + { name => "NSTARS", type => "sum", flag => "-n_stars", dtype => "int" }, +# { name => "?", type => "sum", flag => "-n_extended", dtype => "int" }, +# { name => "?", type => "sum", flag => "-n_cr", dtype => "int" }, + { name => "NASTRO", type => "sum", flag => "-n_astrom", dtype => "int" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +&my_die("Couldn't find input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +## these names are used in ppImage, and thus may be URIs +my $outputImage = $ipprc->filename("PPIMAGE.CHIP", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputMask = $ipprc->filename("PPIMAGE.CHIP.MASK", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputWeight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + +# Run ppImage +unless ($no_op) { + ## XXX can we convert ppImage log and trace to use the filerules to generate consistent names + ## XXX also stats: output should be implied by $outroot + my $command = "$ppImage -file $uri $outroot"; + $command .= " -recipe PPIMAGE $recipe"; + $command .= " -recipe PPSTATS CHIPSTATS"; + $command .= " -stats $outputStats"; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $exp_id, $chip_id, $class_id, $error_code); + } + + ## get the ppImage recipe for this camera and CHIP reduction + $command = "$ppConfigDump -camera $camera -dump-recipe PPIMAGE -recipe PPIMAGE $recipe -"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + ## allow the output images to be optional, depending on the recipe / reduction class + my $outputImageExpect = metadataLookupBool($recipeData, 'CHIP.FITS'); + if ($outputImageExpect) { + &my_die("Couldn't find expected output file: $outputImage\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); + } + + my $outputMaskExpect = metadataLookupBool($recipeData, 'CHIP.MASK.FITS'); + if ($outputMaskExpect) { + &my_die("Couldn't find expected output file: $outputMask\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + } + + my $outputWeightExpect = metadataLookupBool($recipeData, 'CHIP.WEIGHT.FITS'); + if ($outputWeightExpect) { + &my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + } + + &my_die("Couldn't find expected output file: $outputBin1\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1); + &my_die("Couldn't find expected output file: $outputBin2\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2); + &my_die("Couldn't find expected output file: $outputStats\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + + # Get the statistics on the processed image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new(); # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents); + unless ($metadata) { + &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR); + } +} + +# command to update database +my $command = "$chiptool -addprocessedimfile"; +$command .= " -exp_id $exp_id"; +$command .= " -chip_id $chip_id"; +$command .= " -class_id $class_id"; +$command .= " -uri $outputImage"; +$command .= " -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform chiptool -addprocessedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; # Chiptool identifier + my $chip_id = shift; # Chiptool identifier + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $chip_id and defined $class_id and not $no_update) { + my $command = "$chiptool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -chip_id $chip_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -uri $outputImage"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_correct_imfile.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_correct_imfile.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_correct_imfile.pl (revision 22293) @@ -0,0 +1,153 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $exp_id, $class_id, $det_type, $exp_tag, $input_uri, $camera, $dbname, $workdir, $reduction, + $verbose, $no_update, $no_op ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'exp_id|e=s' => \$exp_id, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'exp_tag|=s' => \$exp_tag, + 'input_uri|u=s' => \$input_uri, + 'corr_id|u=s' => \$corr_id, + 'corr_type|u=s' => \$corr_type, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --exp_id --class_id --det_type --exp_tag --input_uri --camera", + -exitval => 3) + unless defined $det_id + and defined $exp_id + and defined $class_id + and defined $det_type + and defined $exp_tag + and defined $input_uri + and defined $camera; + +# XXX this exits with status = 0 on failure +$ipprc->define_camera($camera); + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_PROCESS'); # Recipe name to use + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $dvocorr = can_run('dvoApplyCorr') or (warn "Can't find dvoApplyCorr" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Couldn't find input file: $input_uri\n", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri); + +$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir; + +# detselect -select -det_id $corr_id -class_id $class_id; + +my $outputRoot = $ipprc->file_prepare( "$exp_tag/$exp_tag.detproc.$det_id", $workdir, $input_uri ); +my $outputImage = $ipprc->filename("DVOCORR.OUTPUT", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + +# Run dvoApplyCorr +unless ($no_op) { + my $command = "$dvocorr -file $input_uri $outputRoot"; + $command .= " -corr $corr_uri"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $class_id, $error_code); + } + + &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage); +} + +# command to update database +my $command = "$dettool -addcorrectedimfile"; +$command .= " -det_id $det_id"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -uri $outputImage -path_base $outputRoot"; +$command .= " -dbname $dbname" if defined $dbname; + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addcorrectedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $exp_id = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $exp_id and defined $class_id and not $no_update) { + my $command = "$dettool -addcorrectedimfile"; + $command .= " -det_id $det_id"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_norm_apply.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_norm_apply.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_norm_apply.pl (revision 22293) @@ -0,0 +1,206 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line +my ( $det_id, $iter, $class_id, $value, $input_uri, $camera, $det_type, $outroot, $dbname, $verbose, + $no_update, $no_op ); +GetOptions( + 'det_id|d=s' => \$det_id, # Detrend ID + 'iteration|n=s' => \$iter, # Iteration + 'class_id|i=s' => \$class_id, # Class ID + 'value|v=s' => \$value, # Value to multiple (for normalisation) + 'input_uri|u=s' => \$input_uri, # Input file + 'camera|c=s' => \$camera, # Camera + 'det_type|t=s' => \$det_type, # Detrend type + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database + 'no-op' => \$no_op, # Don't do any operations + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --class_id --value --input_uri --camera --det_type --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $class_id and + defined $value and + defined $input_uri and + defined $camera and + defined $det_type and + defined $outroot; + +$ipprc->define_camera($camera); + +my $RECIPE_PPIMAGE = 'PPIMAGE_N'; # Recipe to use with ppImage +my $RECIPE_PPSTATS = 'CHIPSTATS'; # Recipe to use with ppStats + +my $isMaskType = 0; +if (lc($det_type) eq "mask") { $isMaskType = 1; } +if (lc($det_type) eq "darkmask") { $isMaskType = 1; } +if (lc($det_type) eq "flatmask") { $isMaskType = 1; } + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Couldn't find input file: $input_uri\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $outFile = $isMaskType ? 'PPIMAGE.OUTPUT.DETMASK' : 'PPIMAGE.OUTPUT';; # XXXX something of a hack (too many places to control things...) + +my $output = $ipprc->filename($outFile, $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $b1name = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $b2name = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $statsName = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); + +# Run normalisation +unless ($no_op) { + my $command = "$ppImage -file $input_uri $outroot"; + $command .= " -norm $value -stats $statsName"; + $command .= " -recipe PPIMAGE $RECIPE_PPIMAGE"; + $command .= " -recipe PPSTATS $RECIPE_PPSTATS"; + $command .= " -F PPIMAGE.OUTPUT $outFile" unless $outFile eq 'PPIMAGE.OUTPUT'; + $command .= ' -isfringe' if lc($det_type) eq 'fringe'; + $command .= ' -isdark' if lc($det_type) eq 'dark'; + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $class_id, $error_code); + } + &my_die("Can't find expected output file: $output", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($output); + &my_die("Can't find expected output file: $b1name", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($b1name); + &my_die("Can't find expected output file: $b2name", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($b2name); + &my_die("Can't find expected output file: $statsName", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($statsName); + + # Get the statistics on the normalised image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($statsName) or &my_die("Can't open statistics file $statsName: $!\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + &my_die("Can't open statistics file $statsName: $!\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless defined $statsFile; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents); + unless ($metadata) { + &my_die("Unable to parse metadata config", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + } +} + +# Command to update the database +my $command = "$dettool -addnormalizedimfile"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -class_id $class_id"; +$command .= " -uri $output"; +$command .= " -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addnormalizedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $class_id and not $no_update) { + my $command = "$dettool -addnormalizedimfile"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_norm_calc.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_norm_calc.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_norm_calc.pl (revision 22293) @@ -0,0 +1,233 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run ); +use IPC::Run 0.36 qw( run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse command-line arguments +my ($det_id, $iter, $detType, $outroot, $dbname, $verbose, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, # Detrend id + 'iteration|i=s' => \$iter, # Iteration + 'det_type|t=s' => \$detType, # Detrend type + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do operations + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options --det_id --iteration --det_type --outroot", + -exitval => 3, + ) unless + defined $det_id and + defined $iter and + defined $detType and + defined $outroot; + +use constant STATISTIC => 'bg'; # Background statistic to use from the database + +# Define which detrend types we normalise +use constant NORMALIZE => { + 'bias' => 0, + 'dark' => 0, + 'shutter' => 0, + 'flat' => 1, + 'domeflat' => 1, + 'skyflat' => 1, + 'fringe' => 0, + 'mask' => 0, + 'darkmask' => 0, + 'flatmask' => 0, + }; + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppNormCalc = can_run('ppNormCalc') or (warn "Can't find ppNormCalc" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Unrecognised detrend type: $detType", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless exists NORMALIZE()->{lc($detType)}; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get the list of inputs +my @files; # The input files +{ + my $command = "$dettool -processedimfile -det_id $det_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my @command = split /\s+/, $command; + my ( $stdin, $stdout, $stderr ); # Buffers for running program + print "Running [$command]...\n" if $verbose; + if (not run(\@command, \$stdin, \$stdout, \$stderr)) { + &my_die("Unable to perform dettool -processedimfile on detrend $det_id/$iter: $?", + $det_id, $iter, $PS_EXIT_SYS_ERROR); + } + print $stdout . "\n" if $verbose; + + # Because of the length, need to split into individual metadatas --- it parses SO much quicker! + my @whole = split /\n/, $stdout; + my @single = (); + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ); + &my_die("Unable to parse output from dettool", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless + defined $list; + push @files, @$list; + @single = (); + } + } +} + +my $norms; # MDC with normalisations +if (NORMALIZE()->{lc($detType)} and not $no_op) { + + my %matrix; # Matrix of statistics as a function of exposures and classes + foreach my $file (@files) { + my $exp_id = $file->{'exp_id'}; # Exposure ID + my $class_id = $file->{'class_id'}; # Class ID + my $stat = $file->{STATISTIC()}; # Statistic of interest + + # Create matrix elements + $matrix{$exp_id} = {} if not defined $matrix{$exp_id}; + $matrix{$exp_id}->{$class_id} = $stat; + } + + # Generate the input for ppNormCalc + my $normData; # Normalisation data + foreach my $exp_id (keys %matrix) { + $normData .= "$exp_id\tMETADATA\n"; + foreach my $class_id (keys %{$matrix{$exp_id}}) { + $normData .= "\t" . $class_id . "\tF32\t" . $matrix{$exp_id}->{$class_id} . "\n"; + } + $normData .= "END\n\n"; + } + + # Run ppNormCalc + { + my ( $stdout, $stderr ); # Buffers for running program + my @command = split /\s+/, $ppNormCalc; + print "Running [$ppNormCalc]...\n" if $verbose; + if (not run(\@command, \$normData, \$stdout, \$stderr)) { + &my_die("Unable to perform ppNormCalc: $?", $det_id, $iter, $PS_EXIT_SYS_ERROR); + } + print $stdout . "\n" if $verbose; + + # Parse the output + $norms = $mdcParser->parse($stdout); + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless $norms; + } + +} else { + # It's something that doesn't need normalisation --- just push in a normalisation of 1 + my %classes; # List of unique classes + foreach my $file (@files) { + my $class_id = $file->{'class_id'}; # Class Id + $classes{$class_id} = 1; + } + + foreach my $class_id (keys %classes) { + my %mdValue; # Metadata value for this class id + $mdValue{name} = $class_id; + $mdValue{value} = 1.0; + push @$norms, \%mdValue; + } +} + +my $commandBase = "$dettool -addnormalizedstat"; +$commandBase .= " -det_id $det_id"; +$commandBase .= " -iteration $iter"; +$commandBase .= " -dbname $dbname" if defined $dbname; + +# Process output normalisations +unless ($no_update) { + foreach my $normItem (@$norms) { + + my $className = $normItem->{name}; # Name of component + my $normalisation = $normItem->{value}; # Normalisation for component + + if ($normalisation == 0.0 or lc($normalisation) eq 'nan') { + warn("Class $className has bad normalisation: $normalisation"); + exit($PS_EXIT_SYS_ERROR); + } + + my $command = $commandBase; + $command .= " -class_id $className"; + $command .= " -norm $normalisation"; + + my @command = split /\s+/, $command; + + my ( $stdin, $stdout, $stderr ); # Buffers for running program + print "Running [$command]...\n" if $verbose; + if (not run \@command, \$stdin, \$stdout, \$stderr) { + warn("Unable to perform dettool -addnormstat for $className: $?"); + exit($PS_EXIT_SYS_ERROR); + } + print $stdout . "\n" if $verbose; + } +} else { + print "skipping command: $commandBase\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and not $no_update) { + my $command = "$dettool -addnormalizedstat"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_norm_exp.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_norm_exp.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_norm_exp.pl (revision 22293) @@ -0,0 +1,203 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $iter, $det_type, $camera, $outroot, $dbname, $reduction, $verbose, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration|i=s' => \$iter, + 'camera|c=s' => \$camera, + 'det_type|t=s' => \$det_type, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction|=s' => \$reduction, + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --camera --det_type --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $camera and + defined $det_type and + defined $outroot; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DETREND' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + # { name => "bg_mean_stdev", type => "rms", flag => "-bg_mean_stdev" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -normalizedimfile -det_id $det_id -iteration $iter"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to get list of normalized imfiles from dettool: $error_code", $det_id, $iter, $error_code); + } + + # convert stdout to a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the stats in the metadata + unless ($stats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.\n", $det_id, $iter, $PS_EXIT_PROG_ERROR); + } +} + +my ($list1File, $list1Name) = tempfile( "$camera.$det_type.norm.$det_id.$iter.b1.list.XXXX", UNLINK => 1 ); +my ($list2File, $list2Name) = tempfile( "$camera.$det_type.norm.$det_id.$iter.b2.list.XXXX", UNLINK => 1 ); +foreach my $file (@$files) { + print $list1File ( $ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); + print $list2File ( $ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n"); +} +close $list1File; +close $list2File; + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $jpeg1Name = $ipprc->filename("PPIMAGE.JPEG1", $outroot); # Binned JPEG #1 +my $jpeg2Name = $ipprc->filename("PPIMAGE.JPEG2", $outroot); # Binned JPEG #2 + +unless ($no_op) { + # Make the jpeg for binning 1 + $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1Name); + + # Make the jpeg for binning 2 + $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2Name); + +} + +# command to update the database +$command = "$dettool -addnormalizedexp"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -recip $recipe1,$recipe2"; +$command .= " -path_base $outroot "; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -addnormalizedexp: $error_code", $det_id, $iter, $error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and not $no_update) { + my $command = "$dettool -addprocessedimfile"; + $command .= " -det_id $det_id"; + $command .= " -iter $iter"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_process_exp.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_process_exp.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_process_exp.pl (revision 22293) @@ -0,0 +1,214 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $exp_id, $det_type, $exp_tag, $camera, $outroot, $dbname, $reduction, $verbose, $no_update, + $no_op ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'det_type|t=s' => \$det_type, + 'exp_id|d=s' => \$exp_id, + 'exp_tag|=s' => \$exp_tag, + 'camera|c=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction|=s' => \$reduction, + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --det_type --exp_id --exp_tag --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $det_type and + defined $exp_id and + defined $exp_tag and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DETREND' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_id, $PS_EXIT_PROG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_id, $PS_EXIT_PROG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + # { name => "bg_mean_stdev", type => "rms", flag => "-bg_mean_stdev" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -processedimfile -det_id $det_id -exp_id $exp_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $exp_id, $error_code); + } + + # convert stdout to a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + + # parse the stats in the metadata + unless ($stats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.\n", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + } +} + +# File lists for the two binnings +my ($list1File, $list1Name) = tempfile( "$exp_tag.detproc.$det_id.b1.list.XXXX", UNLINK => 1 ); +my ($list2File, $list2Name) = tempfile( "$exp_tag.detproc.$det_id.b2.list.XXXX", UNLINK => 1 ); +foreach my $file (@$files) { + print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); + print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n"); +} +close $list1File; +close $list2File; + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outroot); # Binned JPEG #1 +my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outroot); # Binned JPEG #2 + +unless ($no_op) { + # Make the jpeg for binning 1 + $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1", $det_id, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1); + + # Make the jpeg for binning 2 + $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2", $det_id, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2); +} + +# Command to update the database +$command = "$dettool -addprocessedexp"; +$command .= " -det_id $det_id"; +$command .= " -exp_id $exp_id"; +$command .= " -recip $recipe1,$recipe2 -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addprocessedexp: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $exp_id = shift; # Exposure tag + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $exp_id and not $no_update) { + my $command = "$dettool -addprocessedexp"; + $command .= " -det_id $det_id"; + $command .= " -exp_id $exp_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_process_imfile.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_process_imfile.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_process_imfile.pl (revision 22293) @@ -0,0 +1,194 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $exp_id, $class_id, $det_type, $exp_tag, $input_uri, $camera, $outroot, $dbname, $reduction, + $verbose, $no_update, $no_op ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'exp_id|e=s' => \$exp_id, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'exp_tag|=s' => \$exp_tag, + 'input_uri|u=s' => \$input_uri, + 'camera|c=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --exp_id --class_id --det_type --exp_tag --input_uri --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $exp_id and + defined $class_id and + defined $det_type and + defined $exp_tag and + defined $input_uri and + defined $camera and + defined $outroot; + +# XXX this exits with status = 0 on failure +$ipprc->define_camera($camera); + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_PROCESS'); # Recipe name to use + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} +$ppImage .= " -dbname $dbname" if defined $dbname; + +&my_die("Couldn't find input file: $input_uri\n", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $outputImage = $ipprc->filename("PPIMAGE.OUTPUT", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR); + +# Run ppImage +unless ($no_op) { + my $command = "$ppImage -file $input_uri $outroot"; + $command .= " -recipe PPIMAGE $recipe"; + $command .= " -recipe PPSTATS CHIPSTATS"; + $command .= " -stats $outputStats"; + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $class_id, $error_code); + } + + &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage); + &my_die("Couldn't find expected output file: $outputStats", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputBin1", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin1); + &my_die("Couldn't find expected output file: $outputBin2", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin2); + + # Get the statistics on the processed image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve("$outputStats") or die "Can't open statistics file $outputStats: $!\n"; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents) + or &my_die("Unable to parse metadata config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # extract the stats from the metadata + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +} + +# command to update database +my $command = "$dettool -addprocessedimfile"; +$command .= " -det_id $det_id"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -recip $reduction"; +$command .= " -uri $outputImage -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addprocessedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $exp_id = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $exp_id and defined $class_id and not $no_update) { + my $command = "$dettool -addprocessedimfile"; + $command .= " -det_id $det_id"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_reject_exp.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_reject_exp.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_reject_exp.pl (revision 22293) @@ -0,0 +1,364 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Metadata::List qw( parse_md_list ); +use Statistics::Descriptive; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my $ITER_LIMIT = 20; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $iter, $det_type, $camera, $outroot, $filter, $dbname, $verbose, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'det_type|t=s' => \$det_type, + 'camera=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'filter=s' => \$filter, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --det_type --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $det_type and + defined $camera and + defined $outroot; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# these stats are used it the rejections but not passed to the database +# there is some duplication with the above, but the calculation time is minimal +my $REJSTATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "ensMeanMean", dtype => "float" }, + { name => "bg", type => "stdev", flag => "ensMeanStdev", dtype => "float" }, + { name => "bg_mean_stdev", type => "mean", flag => "ensMeanStdevMean", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "ensMeanStdevStdev", dtype => "float" }, + { name => "bg_stdev", type => "mean", flag => "ensStdevMean", dtype => "float" }, + { name => "bg_stdev", type => "stdev", flag => "ensStdevStdev", dtype => "float" }, + { name => "bg_skewness", type => "mean", flag => "ensSkewness", dtype => "float" }, + { name => "bg_kurtosis", type => "mean", flag => "ensKurtosis", dtype => "float" }, + + ]; +my $rejstats = PS::IPP::Metadata::Stats->new($REJSTATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($exposures, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + # dettool command to select exp data for this det_run + $command = "$dettool -residexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -residexp: $error_code", $det_id, $iter, $error_code); + } + + # Parse the stdout buffer into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $exposures = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # Parse the statistics for rejections + $rejstats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $PS_EXIT_PROG_ERROR); +} + +# we use the statistics of the ensemble to accept/reject exposurs +my $ensMeanMean = $rejstats->value_for_flag ("ensMeanMean"); +my $ensMeanStdev = $rejstats->value_for_flag ("ensMeanStdev"); +my $ensMeanStdevMean = $rejstats->value_for_flag ("ensMeanStdevMean"); +my $ensMeanStdevStdev = $rejstats->value_for_flag ("ensMeanStdevStdev"); +my $ensStdevMean = $rejstats->value_for_flag ("ensStdevMean"); +my $ensStdevStdev = $rejstats->value_for_flag ("ensStdevStdev"); + +$ipprc->define_camera($camera); +# Rejection thresholds +my $reject_mean = rejection_limit( 'ENSEMBLE.MEAN', $det_type, $filter ); +my $reject_stdev = rejection_limit( 'ENSEMBLE.STDEV', $det_type, $filter ); +my $reject_meanstdev = rejection_limit( 'ENSEMBLE.MEANSTDEV', $det_type, $filter ); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $logName = $outroot . "log"; # Name for log + +my $logFile; +unless ($no_op) { + $logFile = $ipprc->file_create_append( $logName ); + print $logFile "Ensemble mean $ensMeanMean +/- $ensMeanStdevMean, stdev $ensStdevMean\n\n"; +} + +# Go through again to do rejection, and update the database for each exposure +my $numChanges = 0; # Number of exposures with changed status +my $numReject = 0; # Number of exposures rejected +my $firstElement = 1; + +foreach my $exposure (@$exposures) { + my $mean = $exposure->{bg}; # Mean for this exposure + my $stdev = $exposure->{bg_stdev}; # Stdev for this exposure + my $meanStdev = $exposure->{bg_mean_stdev}; # Stdev of Means for this exposure + my $expID = $exposure->{exp_id}; + my $accept = $exposure->{accept}; + my $include = $exposure->{include}; + + &my_die("Unable to find exposure id.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $expID; + &my_die("Unable to find accept.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $accept; + &my_die("Unable to find include.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $include; + + my $reject = 0; # Reject this exposure? + + $command = "$dettool -updateresidexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $expID"; + $command .= " -dbname $dbname" if defined $dbname; + + if (not $accept) { + # Rejected this at an earlier stage + unless ($no_op) { + print $logFile "Rejecting $expID based on earlier determination.\n"; + } + $reject = 1; + goto UPDATE; + } + + # Cop-out if we're not doing any operations + if ($no_op) { + # Make sure something gets rejected (just once!), just so that + # we can trace the full range of the workflow + if ($firstElement and $iter == 0) { + $reject = 1; + } + goto UPDATE; + } + + if ($reject_mean > 0 and $ensMeanStdev > 0) { + my $delta = abs($mean - $ensMeanMean); + if ($delta > ($reject_mean * $ensMeanStdev)) { + print $logFile "Rejecting $expID based on ensemble mean value: "; + $reject = 1; + #goto UPDATE; + } else { + print $logFile "$expID OK against ensemble mean: "; + } + print $logFile "$mean --> $delta vs " . $reject_mean * $ensMeanStdev . "\n"; + } else { + print $logFile "No rejection of $expID for ensemble mean\n"; + } + + if ($reject_stdev > 0 and $ensStdevStdev > 0) { + my $delta = abs($stdev - $ensStdevMean); + if ($delta > ($reject_stdev * $ensStdevStdev)) { + print $logFile "Rejecting $expID based on ensemble stdev: "; + $reject = 1; + #goto UPDATE; + } else { + print $logFile "$expID OK against ensemble stdev: "; + } + print $logFile "$stdev --> $delta sigma vs " . $reject_stdev * $ensStdevStdev . "\n"; + } else { + print $logFile "No rejection of $expID for ensemble stdev\n"; + } + + if ($reject_meanstdev > 0 and $ensMeanStdevStdev > 0) { + my $delta = abs($meanStdev - $ensMeanStdevMean); + if ($delta > ($reject_meanstdev * $ensMeanStdevStdev)) { + print $logFile "Rejecting $expID based on ensemble mean stdev: "; + $reject = 1; + #goto UPDATE; + } else { + print $logFile "$expID OK against ensemble mean stdev: "; + } + print $logFile "$meanStdev --> $delta sigma vs " . $reject_meanstdev * $ensMeanStdevStdev. "\n"; + } else { + print $logFile "No rejection of $expID for ensemble mean stdev\n"; + } + + UPDATE: + if ($reject) { + $command .= ' -reject'; + $numReject++; + } + + # Check for status changes + if ((not $include and not $reject) or ($include and $reject)) { + unless ($no_op) { + print $logFile "Status of $expID has changed.\n"; + } + $numChanges++; + } + + unless ($no_update) { + # Update + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -updateresidexp: $error_code", $det_id, $iter, $error_code); + } + } +} + +# Decide if the current is sufficient to use as a master, and if we can stop iterating +my $master = 1; # This is good enough for a master +my $stop = 1; # Stop iterating + +if ($numChanges > 0) { + $master = 0; + $stop = 0; +} + +# Rejecting everything --- stop before something bad happens! +if ($numReject == scalar @$exposures) { + $master = 0; + $stop = 1; +} + +unless ($no_op) { + print $logFile "Master: $master\n"; + print $logFile "Stop: $stop\n"; + close $logFile; +} + +# attempt to prevent endless, pathological iterations +if ($iter >= $ITER_LIMIT) { + warn("iteration limit reached -- bailing out"); + exit($PS_EXIT_PROG_ERROR); +} + +## add the summary statistics, and request a new iteration if needed +$command = "$dettool -adddetrunsummary"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -accept" if $master; +$command .= " -dbname $dbname" if defined $dbname; +$command .= " -again" unless $stop; +$command .= $stats->cmdflags(); + +# Put results into the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -adddetrunsummary: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and not $no_update) { + my $command = "$dettool -adddetrunsummary"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Retrieve the requested rejection limit, dying unless extant +sub rejection_limit +{ + my $name = shift; # Rejection limit to + my $type = shift; # Type of exposure + my $filter = shift; # Filter + + my $value = $ipprc->rejection( $name, $det_type, $filter ); + if (not defined $value) { + $filter = "(no filter)" unless defined $filter; + die "Unable to determine $name rejection limit for $det_type with $filter.\n"; + } + + return $value; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_reject_imfile.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_reject_imfile.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_reject_imfile.pl (revision 22293) @@ -0,0 +1,607 @@ +#!/usr/bin/env perl + +# this program has two jobs: +# 1: for the given exp_tag, generate the binned & mosaiced JPEG images +# 2: examine the collection of per-imfile statistics and reject. At the moment, +# this program (and the database) only allows rejection at the exposure level, +# not at the imfile level. + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); # tools to run UNIX programs with control over I/O +use PS::IPP::Metadata::Config; # tools to parse the psMetadataConfig files +use PS::IPP::Metadata::Stats; + +use PS::IPP::Metadata::List qw( parse_md_list ); # tools to parse a metadata into a hash list +use Statistics::Descriptive; # tools for calculating basic statistical quantities +use File::Temp qw( tempfile ); # tools to construct temp files + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); # tools to parse the IPP configuration information +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); # option parsing +use Pod::Usage qw( pod2usage ); + +# parse the command-line options +my ( $det_id, $iter, $exp_id, $exp_tag, $det_type, $camera, $filter, $reject, $outroot, $dbname, $reduction, + $verbose, $no_update, $no_op ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'exp_id|e=s' => \$exp_id, + 'exp_tag|=s' => \$exp_tag, + 'det_type|t=s' => \$det_type, + 'camera=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'filter=s' => \$filter, + 'reject' => \$reject, + 'dbname|d=s' => \$dbname, # Database name + 'reduction|=s' => \$reduction, + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --exp_id --exp_tag --det_type --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $exp_id and + defined $exp_tag and + defined $det_type and + defined $camera and + defined $outroot; + +# load IPP config information for the specified camera +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DETREND' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_RESID_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_RESID_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "bg_skewness", type => "mean", flag => "-bg_skewness", dtype => "float" }, + { name => "bg_kurtosis", type => "mean", flag => "-bg_kurtosis", dtype => "float" }, + { name => "bin_stdev", type => "rms", flag => "-bin_stdev", dtype => "float" }, + { name => "fringe_0", type => "mean", flag => "-fringe_0", dtype => "float" }, + { name => "fringe_1", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "fringe_0", type => "stdev", flag => "-fringe_2", dtype => "float" }, + { name => "user_1", type => "mean", flag => "-user_1", dtype => "float" }, # fringe residual + { name => "user_2", type => "rms", flag => "-user_2", dtype => "float" }, # fringe residual + { name => "user_1", type => "stdev", flag => "-user_3", dtype => "float" }, # fringe residual + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of imfile files +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + # dettool command to select imfile data for this exp_id + $command = "$dettool -residimfile"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $exp_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -residimfile: $error_code\n"); + exit($error_code); + } + # XXX report an error message if stdout_buf is empty + + # Parse the stdout buffer into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR); +} + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $jpeg1Name = $ipprc->filename("PPIMAGE.JPEG1", $outroot); # Binned JPEG #1 +my $jpeg2Name = $ipprc->filename("PPIMAGE.JPEG2", $outroot); # Binned JPEG #2 +my $logName = $ipprc->filename("LOG.EXP", $outroot); # Name for log + +my $logFile; +unless ($no_op) { + $logFile = $ipprc->file_create_append( $logName ); +} + +# XXX in debug mode, unlink = 0 +my ($list1File, $list1Name) = tempfile( "$exp_tag.detresid.$det_id.$iter.b1.list.XXXX", UNLINK => 1 ); +my ($list2File, $list2Name) = tempfile( "$exp_tag.detresid.$det_id.$iter.b2.list.XXXX", UNLINK => 1 ); +foreach my $file (@$files) { + print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); + print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n"); +} +close $list1File; +close $list2File; + + +# build the JPEG images +unless ($no_op) { + # Make the jpeg for binning 1 + $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1Name); + + # Make the jpeg for binning 2 + $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2Name); +} + +#### measure stats and reject if needed #### + +# Rejection thresholds & related data +my $expected = rejection_limit( 'EXPECTED', $det_type, $filter ); +my $reject_imfile_mean = rejection_limit( 'IMFILE.MEAN', $det_type, $filter ); +my $reject_imfile_flux = rejection_limit( 'IMFILE.FLUX', $det_type, $filter ); +my $reject_imfile_stdev = rejection_limit( 'IMFILE.STDEV', $det_type, $filter ); +my $reject_imfile_skewness = rejection_limit( 'IMFILE.SKEWNESS', $det_type, $filter ); +my $reject_imfile_kurtosis = rejection_limit( 'IMFILE.KURTOSIS', $det_type, $filter ); +my $reject_imfile_meanstdev = rejection_limit( 'IMFILE.MEANSTDEV', $det_type, $filter ); +my $reject_imfile_snr = rejection_limit( 'IMFILE.SNR', $det_type, $filter ); +my $reject_imfile_bin_stdev = rejection_limit( 'IMFILE.BIN.STDEV', $det_type, $filter ); +my $reject_imfile_bin_snr = rejection_limit( 'IMFILE.BIN.SNR', $det_type, $filter ); +my $reject_exp_mean = rejection_limit( 'EXP.MEAN', $det_type, $filter ); +my $reject_exp_flux = rejection_limit( 'EXP.FLUX', $det_type, $filter ); +my $reject_exp_stdev = rejection_limit( 'EXP.STDEV', $det_type, $filter ); +my $reject_exp_meanstdev = rejection_limit( 'EXP.MEANSTDEV', $det_type, $filter ); +my $reject_exp_snr = rejection_limit( 'EXP.SNR', $det_type, $filter ); +my $reject_exp_bin_stdev = rejection_limit( 'EXP.BIN.STDEV', $det_type, $filter ); +my $reject_exp_bin_snr = rejection_limit( 'EXP.BIN.SNR', $det_type, $filter ); + +# storage array +my @fluxes; + +foreach my $file (@$files) { + my $name = $file->{class_id}; + my $mean = $file->{bg}; # Mean for this imfile + my $stdev = $file->{bg_stdev}; # Stdev for this imfile + my $skewness = $file->{bg_skewness}; # Skewness for this imfile + my $kurtosis = $file->{bg_kurtosis}; # Kurtosis for this imfile + my $meanStdev = $file->{bg_mean_stdev}; # Stdev of Means for this imfile + my $binStdev = $file->{bin_stdev}; # Binned Stdev for this imfile + + # calculate and save the fluxes + my $flux; + if ($file->{exp_time} == 0.0) { + $flux = $mean; + } else { + $flux = $mean / $file->{exp_time}; + } + push @fluxes, $flux; + + $mean -= $expected; + + last if $no_op; + + # reject exposure if, for any imfiles, the mean residual counts + # deviate from the expected value by more than N sigma, (sigma = + # total pixel variance). this test is sensible for images which + # should have a predictable nominal residual mean count value (eg, + # 0.0 for a bias). in general, use with ADDITIVE components + if ($reject_imfile_mean > 0) { + if (abs($mean) > $reject_imfile_mean * $stdev) { + print $logFile "Rejecting exposure based on bad imfile mean for $name: "; + $reject = 1; + } else { + print $logFile "Imfile mean for $name meets requirements: "; + } + print $logFile "$mean vs $reject_imfile_mean * $stdev\n"; + } else { + print $logFile "No rejection on imfile mean for $name\n"; + } + + # reject exposure if, for any imfiles, the mean residual flux + # deviates from the expected value by more than N sigma, (sigma = + # total pixel variance). this test is sensible for images which + # should have a predictable nominal residual flux value (eg, 0.0 + # for a bias). in general, use with ADDITIVE components + if ($reject_imfile_flux > 0) { + if (abs($flux) > $reject_imfile_flux) { + print $logFile "Rejecting exposure based on bad imfile flux for $name: "; + $reject = 1; + } else { + print $logFile "Imfile flux for $name meets requirements: "; + } + print $logFile "$flux vs $reject_imfile_flux\n"; + } else { + print $logFile "No rejection on imfile flux for $name\n"; + } + + # reject exposure if, for any imfiles, the total pixel variance is + # larger than the limit + if ($reject_imfile_stdev > 0) { + if ($stdev > $reject_imfile_stdev) { + print $logFile "Rejecting exposure based on bad imfile stdev for $name: "; + $reject = 1; + } else { + print $logFile "Imfile stdev for $name meets requirements: "; + } + print $logFile "$stdev vs $reject_imfile_stdev\n"; + + } else { + print $logFile "No rejection on imfile stdev for $name\n"; + } + + # reject exposure if, for any imfiles, the skewness is + # larger than the limit + if ($reject_imfile_skewness > 0) { + if ($stdev > $reject_imfile_skewness) { + print $logFile "Rejecting exposure based on bad imfile skewness for $name: "; + $reject = 1; + } else { + print $logFile "Imfile skewness for $name meets requirements: "; + } + print $logFile "$skewness vs $reject_imfile_skewness\n"; + + } else { + print $logFile "No rejection on imfile skewness for $name\n"; + } + + # reject exposure if, for any imfiles, the kurtosis is + # larger than the limit + if ($reject_imfile_kurtosis > 0) { + if ($stdev > $reject_imfile_kurtosis) { + print $logFile "Rejecting exposure based on bad imfile kurtosis for $name: "; + $reject = 1; + } else { + print $logFile "Imfile kurtosis for $name meets requirements: "; + } + print $logFile "$kurtosis vs $reject_imfile_kurtosis\n"; + + } else { + print $logFile "No rejection on imfile kurtosis for $name\n"; + } + + # reject exposure if, for any imfiles, the variance of the imfile + # component means is larger than the limit + if ($reject_imfile_meanstdev > 0) { + if ($meanStdev > $reject_imfile_meanstdev) { + print $logFile "Rejecting exposure based on bad imfile mean stdev for $name: "; + $reject = 1; + } else { + print $logFile "Imfile mean stdev for $name meets requirements: "; + } + print $logFile "$meanStdev vs $reject_imfile_meanstdev\n"; + } else { + print $logFile "No rejection on imfile mean stdev for $name\n"; + } + + # reject exposure if, for any imfiles, the signal-to-noise (ie, + # the mean counts / total pixel variance) of the imfile component + # means are less than the limit. this test is sensible for images + # which have finite residual flux such as a flat-field image. + if ($reject_imfile_snr > 0) { + if ($mean < $stdev * $reject_imfile_snr) { + print $logFile "Rejecting exposure based on bad imfile S/N for $name: "; + $reject = 1; + } else { + print $logFile "Imfile S/N for $name meets requirements: "; + } + print $logFile "mean: $mean vs stdev*SNlimit: " . $stdev * $reject_imfile_snr . "\n"; + } else { + print $logFile "No rejection on imfile S/N for $name\n"; + } + + # reject exposure if, for any imfiles, the signal-to-noise of the + # imfile component means, based on the stdev of the binned image + # is less than the limit. this test is sensible for images which + # have finite residual flux such as a flat-field image. + if ($reject_imfile_bin_stdev > 0) { + if ($binStdev > $reject_imfile_bin_stdev) { + print $logFile "Rejecting exposure based on bad imfile binned stdev for $name: "; + $reject = 1; + } else { + print $logFile "Imfile binned stdev for $name meets requirements: "; + } + print $logFile "$binStdev vs $reject_imfile_bin_stdev\n"; + } else { + print $logFile "No rejection on imfile binned stdev for $name\n"; + } + if ($reject_imfile_bin_snr > 0) { + if ($mean < $binStdev * $reject_imfile_bin_snr) { + print $logFile "Rejecting exposure based on bad imfile binned S/N for $name: "; + $reject = 1; + } else { + print $logFile "Imfile binned S/N for $name meets requirements: "; + } + print $logFile "mean: $mean vs binStdev*SNlimit: " . $binStdev * $reject_imfile_bin_snr . "\n"; + } else { + print $logFile "No rejection on imfile binned S/N for $name\n"; + } +} + +# basic ensemble stats +my $mean = $stats->value_for_flag ("-bg"); +my $meanStdev = $stats->value_for_flag ("-bg_mean_stdev"); +my $stdev = $stats->value_for_flag ("-bg_stdev"); +my $binStdev = $stats->value_for_flag ("-bin_stdev"); +my $fringe_mean = $stats->value_for_flag ("-fringe_0"); +my $fringe_err = $stats->value_for_flag ("-fringe_1"); +my $fringe_mean_stdev = $stats->value_for_flag ("-fringe_2"); +my $dfringe_mean = $stats->value_for_flag ("-fringe_resid_0"); +my $dfringe_err = $stats->value_for_flag ("-fringe_resid_1"); +my $dfringe_mean_stdev = $stats->value_for_flag ("-fringe_resid_2"); + +# other stats (flux depends on bg and exp_time) +my $fluxStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for means +$fluxStats->add_data(@fluxes); +my $flux = $fluxStats->mean(); # Mean of the imfile means + +# other stats +my $exp_snr = 0.0; +if ($stdev > 0) { $exp_snr = $mean / $stdev; } + +## Reject based on the exposure ensemble stats +# reject if the exposure ensemble mean is deviant +unless ($no_op) { + print $logFile "Exposure mean $mean, stdev $stdev, mean stdev $meanStdev, exp s/n: $exp_snr\n"; + + # reject exposure if the ensemble mean residual counts deviate + # from the expected value by more than N sigma, (sigma = total + # pixel variance). this test is sensible for images which should + # have a predictable nominal residual mean count value (eg, 0.0 + # for a bias). in general, use with ADDITIVE components + if ($reject_exp_mean > 0) { + if (abs($mean) > $reject_exp_mean * $stdev) { + print $logFile "Rejecting exposure based on bad mean counts: "; + $reject = 1; + } else { + print $logFile "Exposure mean counts meets requirements: "; + } + print $logFile "mean: $mean, stdev: $stdev (limit is: " . $reject_exp_mean * $stdev . ") \n"; + } else { + print $logFile "No rejection for exp mean\n"; + } + + # reject exposure if, for any imfiles, the mean residual flux + # deviates from the expected value by more than N sigma, (sigma = + # total pixel variance). this test is sensible for images which + # should have a predictable nominal residual flux value (eg, 0.0 + # for a bias). in general, use with ADDITIVE components + if ($reject_exp_flux > 0) { + if (abs($flux) > $reject_exp_flux * $stdev) { + print $logFile "Rejecting exposure based on bad mean flux: "; + $reject = 1; + } else { + print $logFile "Exposure mean flux meets requirements: "; + } + print $logFile "flux: $flux, stdev: $stdev (limit is: " . $reject_exp_flux * $stdev . ")\n"; + } else { + print $logFile "No rejection for exp mean\n"; + } + + # reject exposure if the total pixel variance is larger than the + # limit + if ($reject_exp_stdev > 0) { + if ($stdev > $reject_exp_stdev) { + print $logFile "Rejecting exposure based on bad stdev: "; + $reject = 1; + } else { + print $logFile "Exposure stdev meets requirements: "; + } + print $logFile "$stdev vs $reject_exp_stdev\n"; + } else { + print $logFile "No rejection for exp stdev\n"; + } + + # reject exposure if the variance of the imfile means is larger + # than the limit + if ($reject_exp_meanstdev > 0) { + if ($meanStdev > $reject_exp_meanstdev) { + print $logFile "Rejecting exposure based on bad mean stdev: "; + $reject = 1; + } else { + print $logFile "Exposure mean stdev meets requirements: "; + } + print $logFile "$meanStdev vs $reject_exp_meanstdev\n"; + } else { + print $logFile "No rejection for exp mean stdev\n"; + } + + # reject if the signal-to-noise is insufficient + if ($reject_exp_snr > 0) { + if (abs($mean) < abs($stdev * $reject_exp_snr)) { + print $logFile "Rejecting exposure based on poor S/N: \n"; + $reject = 1; + } else { + print $logFile "Exposure S/N meets requirements: \n"; + } + print $logFile "signal: $mean vs noise: $stdev (s/n limit is: $reject_exp_snr)\n"; + } else { + print $logFile "No rejection for exp S/N\n"; + } + # reject if the exposure ensemble stdev is deviant + if ($reject_exp_bin_stdev > 0) { + if ($binStdev > $reject_exp_bin_stdev) { + print $logFile "Rejecting exposure based on bad binned stdev: "; + $reject = 1; + } else { + print $logFile "Exposure binned stdev meets requirements: "; + } + print $logFile "$stdev vs $reject_exp_bin_stdev\n"; + } else { + print $logFile "No rejection for exp binned stdev\n"; + } + # reject if the signal-to-noise is insufficient + if ($reject_exp_bin_snr > 0) { + if (abs($mean) < abs($binStdev * $reject_exp_bin_snr)) { + print $logFile "Rejecting exposure based on poor binned S/N: \n"; + $reject = 1; + } else { + print $logFile "Exposure binned S/N meets requirements: \n"; + } + print $logFile "signal: $mean vs noise: $binStdev (s/n limit is: $reject_exp_bin_snr)\n"; + } else { + print $logFile "No rejection for exp binned S/N\n"; + } + + close $logFile; +} + +$command = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_id $exp_id"; +$command .= " -recip $recipe1,$recipe2 -path_base $outroot "; +$command .= ' -reject' if $reject; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addresidexp: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exp_id = shift; # Exposure tag + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $exp_id and not $no_update) { + my $command = "$dettool -addresidexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $exp_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Retrieve the requested rejection limit, dying if not extant +sub rejection_limit +{ + my $name = shift; # Rejection limit to + my $type = shift; # Type of exposure + my $filter = shift; # Filter + + my $value = $ipprc->rejection( $name, $det_type, $filter ); + if (not defined $value) { + $filter = "(no filter)" if not defined $filter; + die "Unable to determine $name rejection limit for $det_type with $filter.\n"; + } + + return $value; +} + +__END__ + +#### + +## this function is not well though out. it loads a collection of +## values (background mean and stdev) for a collection of imfile +## subcomponents. it then measures the mean background, the +## background stdev (rms of ensemble stdevs), and the background mean +## stdev (rms of the means). it then tries to reject based on the +## following criteria: +## (background / stdev > limit) for a single component +## ---> this will ACCEPT for a poor images (large stdev) +## (stdev > limit) +## (mean background / stdev > limit) same problem +## (mean stdev > limit) + +### I would suggest the following: +## 1) calculate the ensemble mean and stdev +## 2) for each component, is the (back - mean) / mean stdev > limit? +## (ie, is it an outlier?) +## 3) is the component stdev > limit? (absolute test) +## 4) calculate the background stdev for the ensemble (excluding rejects) +## 5) is the mean stdev > limit (poor images) +## 6) calculate the ensemble stdev (rms) of remaining (excluding rejects) +## 7) is the ensemble stdev > limit + +## for flats, the value used for stdev should be the fractional stdev +## (stdev/mean) and no rejection should be performed on the basis of +## the mean value, but could still be done on the mean stdev + +## the same logical cuts above can be applied to components in an +## imfile, imfiles in an exposure, and exposures in a stack + +### this function needs to avoid div by zero: compare Signal vs Noise*limit Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_resid.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_resid.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_resid.pl (revision 22293) @@ -0,0 +1,260 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $iter, $exp_id, $exp_tag, $class_id, $det_type, $detrend, $input_uri, $camera, $mode, $outroot, + $dbname, $reduction, $verbose, $no_update, $no_op ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'exp_id|e=s' => \$exp_id, + 'exp_tag|=s' => \$exp_tag, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'detrend=s' => \$detrend, + 'input_uri|u=s' => \$input_uri, + 'camera|c=s' => \$camera, + 'mode|m=s' => \$mode, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --exp_id --exp_tag --class_id --det_type --camera --input_uri --mode --detrend --outroot (not for 'verify' mode)", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $exp_id and + defined $exp_tag and + defined $class_id and + defined $det_type and + defined $input_uri and + defined $camera and + defined $mode and + defined $outroot and + (defined $detrend or lc($mode) eq 'verify'); + +$ipprc->define_camera($camera); + +# Recipes to use as a function of detrend type and mode +$reduction = 'DETREND' unless defined $reduction; +my $recipe; # Name of recipe to use +if ($mode eq 'master') { + $recipe = uc($det_type) . '_RESID'; +} elsif ($mode eq 'verify') { + $recipe = uc($det_type) . '_VERIFY'; +} else { + &my_die("Unrecognised mode: $mode", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +} +$recipe = $ipprc->reduction($reduction, $recipe); + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "SAMPLE_SKEWNESS", type => "mean", flag => "-bg_skewness", dtype => "float" }, + { name => "SAMPLE_KURTOSIS", type => "mean", flag => "-bg_kurtosis", dtype => "float" }, + { name => "FRINGE_0", type => "mean", flag => "-fringe_0", dtype => "float" }, + { name => "FRINGE_ERR_0", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "FRINGE_0", type => "stdev", flag => "-fringe_2", dtype => "float" }, + { name => "FRINGE_RESID_0", type => "mean", flag => "-fringe_resid_0", dtype => "float" }, + { name => "FRINGE_RESID_ERR_0", type => "rms", flag => "-fringe_resid_1", dtype => "float" }, + { name => "FRINGE_RESID_0", type => "stdev", flag => "-fringe_resid_2", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +my $BINNED_STATS = + [ + { name => "ROBUST_STDEV", type => "rms", flag => "-bin_stdev" }, + ]; +my $binnedStats = PS::IPP::Metadata::Stats->new($BINNED_STATS); # Stats parser + +# Flags to specify the particular detrend to use +use constant DETRENDS => { + 'bias' => '-bias', # Specify the bias frame + 'dark' => '-dark', # Specify the dark frame + 'shutter' => '-shutter', # Specify the shutter frame + 'flat' => '-flat', # Specify the flat frame + 'domeflat' => '-flat', # Specify the flat frame + 'skyflat' => '-flat', # Specify the flat frame + 'fringe' => '-fringe', # Specify the fringe frame + 'mask' => '-mask', # Specify the mask frame + 'darkmask' => '-mask', # Specify the mask frame + 'flatmask' => '-mask', # Specify the mask frame +}; + +use constant DELETE_STATS => 0; # Delete the statistics file when done? + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppStats = can_run('ppStats') or (warn "Can't find ppStats" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} +$ppImage .= " -dbname $dbname" if defined $dbname; + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $outputName = $ipprc->filename("PPIMAGE.OUTPUT", $outroot, $class_id); +my $bin1Name = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id); +my $bin2Name = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id); +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id); +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id); + +# Run ppImage & ppStats +unless ($no_op) { + my $command = "$ppImage -file $input_uri $outroot"; + $command .= " -recipe PPIMAGE $recipe"; + $command .= " -recipe PPSTATS RESIDUAL"; + $command .= " -stats $outputStats"; + $command .= " -tracedest $traceDest -log $logDest"; + + # Detrend to use in processing + if (lc($mode) ne 'verify') { + my $detFlag = DETRENDS->{lc($det_type)}; + &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR) unless defined $detFlag; + $command .= " $detFlag $detrend"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputName); + &my_die("Couldn't find expected output file: $outputStats", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $bin1Name", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin1Name); + &my_die("Couldn't find expected output file: $bin2Name", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin2Name); + + # Load the raw output stats file + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # Parse the stats file contents into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents) or &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # run ppStats on the binned image + $command = "$ppStats -recipe PPSTATS RESIDUAL $bin2Name"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStats: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code); + } + + # Parse the output contents into a metadata + my $binnedMetadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata output", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # parse the binned image statistics + $binnedStats->parse($binnedMetadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +} + +# Command to update the database +my $command = "$dettool -addresidimfile"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -recip $recipe"; +$command .= " -uri $outputName"; +$command .= " -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); +$command .= $binnedStats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addresidimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exp_id = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $exp_id and not $no_update) { + my $command = "$dettool -addresidimfile"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_stack.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_stack.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/detrend_stack.pl (revision 22293) @@ -0,0 +1,212 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $iter, $class_id, $det_type, $camera, $outroot, $dbname, $reduction, $verbose, $no_update, + $no_op ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'camera|c=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class for processing + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --class_id --det_type --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $class_id and + defined $det_type and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_STACK'); # Recipe name to use + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + # { name => "bg_mean_stdev", type => "rms", flag => "-bg_mean_stdev" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppMerge = can_run('ppMerge') or (warn "Can't find ppMerge" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of files to stack +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -processedimfile -included"; + $command .= " -det_id $det_id"; + $command .= " -class_id $class_id"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $iter, $class_id, $error_code); + } + + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); +} + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $outputStack = $ipprc->filename("PPMERGE.OUTPUT", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Output name +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Statistics name +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Trace messages +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Log messages + + +$command = "$ppMerge $outputStack"; # Command to run +foreach my $file (@$files) { + $command .= ' ' . $file->{uri}; +} +$command .= " -recipe PPMERGE $recipe"; +$command .= ' -type ' . uc($det_type); # Type of stacking to perform +$command .= " -stats $outputStats"; # Statistics output filename +$command .= " -recipe PPSTATS CHIPSTATS"; +$command .= " -tracedest $traceDest -log $logDest"; + +# Stack the files +unless ($no_op) { + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppMerge: $error_code", $det_id, $iter, $class_id, $error_code); + } + &my_die("Unable to find expected output file: $outputStack\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStack); + &my_die("Unable to find expected output file: $outputStats\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + + # Get the statistics on the stacked image + open(my $statsFile, $ipprc->file_resolve("$outputStats")) or + &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + my $contents = do { local $/; <$statsFile> }; # Contents of file + close($statsFile); + + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse($contents) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + + $stats->parse($metadata) or + &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); +} + +# Command to update the database +$command = "$dettool -addstacked"; +$command .= " -det_id $det_id -iteration $iter"; +$command .= " -class_id $class_id"; +$command .= " -uri $outputStack"; +$command .= " -recip $recipe"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the resultant into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addstacked: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $class_id and not $no_update) { + my $command = "$dettool -addstacked"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/diff_skycell.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/diff_skycell.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/diff_skycell.pl (revision 22293) @@ -0,0 +1,267 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($diff_id, $dbname, $outroot, $verbose, $no_update, $no_op); +GetOptions( + 'diff_id|d=s' => \$diff_id, # Diff identifier + 'dbname|d=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --diff_id", + -exitval => 3, + ) unless defined $diff_id + and defined $outroot; + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC DIFFTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, +# { name => "DT_DIFF", type => "sum", flag => "-dtime_diff", dtype => "float" }, + { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $ppSub = can_run('ppSub') or (warn "Can't find ppSub" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of components for subtraction +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$difftool -inputskyfile -diff_id $diff_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -inputskyfile: $error_code", $diff_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $diff_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $diff_id, $PS_EXIT_PROG_ERROR); +} + +&my_die("Subtraction list contains more than two elements", $diff_id, $PS_EXIT_SYS_ERROR) unless + scalar @$files == 2; + +# Identify the input and the template +my ($input, $inputMask, $inputWeight, $inputPath, $inputPSF); # Input files and path +my ($template, $templateMask, $templateWeight, $templatePath, $templateSources); # Template files and path +my $tess_id; # Tesselation identifier +my $skycell_id; # Skycell identifier +my $camera; # Camera +foreach my $file (@$files) { + if (defined $file->{template} and $file->{template}) { + $template = $file->{uri}; + $templatePath = $file->{path_base}; + if ($file->{warp_id} == 0) { + $templateMask = "PPSTACK.OUTPUT.MASK"; + $templateWeight = "PPSTACK.OUTPUT.WEIGHT"; + $templateSources = "PSPHOT.OUTPUT"; + } else { + $templateMask = "PSWARP.OUTPUT.MASK"; + $templateWeight = "PSWARP.OUTPUT.WEIGHT"; + $templateSources = "PSWARP.OUTPUT.SOURCES"; + } + } else { + $input = $file->{uri}; + $inputPath = $file->{path_base}; + if ($file->{warp_id} == 0) { + $inputMask = "PPSTACK.OUTPUT.MASK"; + $inputWeight = "PPSTACK.OUTPUT.WEIGHT"; + $inputPSF = "PSPHOT.PSF.SAVE"; + } else { + $inputMask = "PSWARP.OUTPUT.MASK"; + $inputWeight = "PSWARP.OUTPUT.WEIGHT"; + $inputPSF = "PSPHOT.PSF.SAVE"; + } + } + if (defined $tess_id) { + &my_die("Tesselation identifiers don't match", $diff_id, $PS_EXIT_SYS_ERROR) unless + $file->{tess_id} eq $tess_id; + } else { + $tess_id = $file->{tess_id}; + } + if (defined $skycell_id) { + &my_die("Skycell identifiers don't match", $diff_id, $PS_EXIT_SYS_ERROR) unless + $file->{skycell_id} eq $skycell_id; + } else { + $skycell_id = $file->{skycell_id}; + } + if (defined $camera) { + &my_die("Cameras don't match", $diff_id, $PS_EXIT_SYS_ERROR) unless $file->{camera} eq $camera; + } else { + $camera = $file->{camera}; + } + +} + +&my_die("Unable to identify template", $diff_id, $PS_EXIT_SYS_ERROR) unless defined $template; +&my_die("Unable to identify input", $diff_id, $PS_EXIT_SYS_ERROR) unless defined $input; +&my_die("Unable to identify camera", $diff_id, $PS_EXIT_SYS_ERROR) unless defined $camera; +$ipprc->define_camera($camera); + +$templateMask = $ipprc->filename($templateMask, $templatePath); +$inputMask = $ipprc->filename($inputMask, $inputPath); +$templateWeight = $ipprc->filename($templateWeight, $templatePath); +$inputWeight = $ipprc->filename($inputWeight, $inputPath); +$inputPSF = $ipprc->filename($inputPSF, $inputPath); +$templateSources = $ipprc->filename($templateSources, $templatePath); + +&my_die("Couldn't find input: $template", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($template); +&my_die("Couldn't find input: $templateMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateMask); +&my_die("Couldn't find input: $templateWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateWeight); +&my_die("Couldn't find input: $input", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input); +&my_die("Couldn't find input: $inputMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask); +&my_die("Couldn't find input: $inputWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputWeight); +&my_die("Couldn't find input: $inputPSF", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputPSF); +&my_die("Couldn't find input: $templateSources", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateSources); + +# Get the output filenames +my $outputName = $ipprc->filename("PPSUB.OUTPUT", $outroot); +my $outputMask = $ipprc->filename("PPSUB.OUTPUT.MASK", $outroot); +my $outputWeight = $ipprc->filename("PPSUB.OUTPUT.WEIGHT", $outroot); +my $outputSources = $ipprc->filename("PSPHOT.OUTPUT", $outroot); +#my $bin1Name = $ipprc->filename("PPSUB.BIN1", $outroot); +#my $bin2Name = $ipprc->filename("PPSUB.BIN2", $outroot); +my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); +my $logDest = $ipprc->filename("LOG.EXP", $outroot); + +# Perform subtraction +unless ($no_op) { + my $command = "$ppSub $input $template $outroot"; + $command .= " -inmask $inputMask"; + $command .= " -refmask $templateMask"; + $command .= " -inweight $inputWeight"; + $command .= " -refweight $templateWeight"; + $command .= " -stats $outputStats"; + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -sources $templateSources"; + $command .= " -psf $inputPSF"; + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppSub: $error_code", $diff_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + &my_die("Couldn't find expected output file: $outputMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + &my_die("Couldn't find expected output file: $outputSources", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); +# &my_die("Couldn't find expected output file: $bin1Name", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name); +# &my_die("Couldn't find expected output file: $bin2Name", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name); + &my_die("Couldn't find expected output file: $outputStats", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + + # Get the statistics on the residual image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $diff_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $metadata = $mdcParser->parse(join "", @contents) or + &my_die("Unable to parse metadata config doc", $diff_id, $PS_EXIT_PROG_ERROR); + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $diff_id, $PS_EXIT_PROG_ERROR); +} + +unless ($no_update) { + + # Add the subtraction result + { + my $command = "$difftool -adddiffskyfile -diff_id $diff_id -uri $outputName -path_base $outroot"; + $command .= $stats->cmdflags(); + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -adddiffskyfile: $error_code", $diff_id, $error_code); + } + } + + # Register the run as completed + { + my $command = "$difftool -updaterun -diff_id $diff_id -state stop"; # Command to run difftool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform difftool -updaterun: $error_code\n"); + exit($error_code); + } + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $diff_id = shift; # Diff identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if (defined $diff_id and not $no_update) { + my $command = "$difftool -adddiffskyfile -diff_id $diff_id -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ds9_cmf_regions.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ds9_cmf_regions.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ds9_cmf_regions.pl (revision 22293) @@ -0,0 +1,127 @@ +#!/usr/bin/env perl + +use strict; +use Astro::FITS::CFITSIO qw( :constants ); +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use Data::Dumper; + +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +my $xpaset = `which xpaset`; +my $xpaget = `which xpaget`; + +die "Unable to find xpaget and xpaset.\n" unless ($xpaset =~ /\S+/ and $xpaget =~ /\S+/); + + +my ( $filename, # Filename containing photometry + $extname, # Extension name containing photometry + $frame, # Frame number in ds9 + $colour, # Region colour + $radius, # Radius for circle + ); + +# Defaults +$colour = "red"; +$radius = 5; + +GetOptions( + 'file=s' => \$filename, + 'ext=s' => \$extname, + 'frame=s' => \$frame, + 'colour=s' => \$colour, + 'radius=f' => \$radius, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --file --ext", + -exitval => 3) + unless defined $filename + and defined $extname; + +my $status; # Status of FITSIO calls +my $fits = Astro::FITS::CFITSIO::open_file( $filename, READONLY, $status ); # FITS file handle +check_fitsio($status); +$fits->movnam_hdu(BINARY_TBL, $extname, 0, $status) and check_fitsio($status); +my $numRows; # Number of rows in table +$fits->get_num_rows($numRows, $status) and check_fitsio($status); + +my ($xCol, $yCol); # Column numbers for x and y +$fits->get_colnum(0, 'X_PSF', $xCol, $status) and check_fitsio($status); +$fits->get_colnum(0, 'Y_PSF', $yCol, $status) and check_fitsio($status); + +my ($xType, $yType); # Types for x and y +$fits->get_coltype($xCol, $xType, undef, undef, $status) and check_fitsio($status); +$fits->get_coltype($yCol, $yType, undef, undef, $status) and check_fitsio($status); + +my ($x, $y); # Coordinates read from table +$fits->read_col($xType, $xCol, 1, 1, $numRows, 0, $x, undef, $status) and check_fitsio($status); +$fits->read_col($yType, $yCol, 1, 1, $numRows, 0, $y, undef, $status) and check_fitsio($status); +$fits->close_file($status); + +my ($coordFile, $coordName) = tempfile( "/tmp/ds9_cmf_regions.XXXX", UNLINK => 1 ); +for (my $i = 0; $i < $numRows; $i++) { + print $coordFile "image; circle(" . ($$x[$i] + 1) . ',' . ($$y[$i] + 1) . + ",$radius \# color = $colour\n"; +} +close $coordFile; + +my @settings = settings_save("regions format", + "regions system"); # Settings to save + +xpaset("frame $frame") if defined $frame; +xpaset("regions format ds9"); +xpaset("regions system image"); +xpaset("regions load $coordName"); + +xpaset(@settings); + +print "Plotted $numRows sources.\n"; + +### Pau. + + + + +# From Astro::FITS::CFITSIO demo +sub check_fitsio +{ + my $status = shift; # Status of FITSIO calls + + if ($status != 0) { + my $msg; # Message to output + Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); + die "CFITSIO error: $msg\n"; + } +} + +# Save specified settings +sub settings_save +{ + my @settings; # Values of settings + foreach my $setting (@_) { + my @values = xpaget($setting); + push @settings, $setting . ' ' . shift @values; + } + return @settings; +} + + +# XPA subroutines courtesy Derek Fox +sub xpaset { + foreach my $cmd (@_) { + system("xpaset -p ds9 $cmd"); + } +} +sub xpaget { + my @out; + foreach my $cmd (@_) { + my $output = `xpaget ds9 $cmd`; + push @out, $output; + } + return @out; +} + + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/flatcorr_init.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/flatcorr_init.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/flatcorr_init.pl (revision 22293) @@ -0,0 +1,221 @@ +#!/usr/bin/env perl + +## USAGE:flatcorr_init.pl +## given dbname, dvodb, filter, time range, camera, telescope, etc? +## select matching images and register as a new flatcorr run + +## should this be implemented as a ippTool program? +## this is a lot like dettool -definebyquery +## flatcorr -definebyquery +## this should queue the selected images with a mode that stops at camera +## ?? how do we block this batch from running through warp ?? + +## we also need the test / query tool: +## flatcorr -pendingcorr +## examine the chip & camera stages and check for all images to have completed (successfully or not) + +## allow a certain fraction of failures... + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dvo_id, $catdir, $region, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'dvo_id|i=s' => \$dvo_id, + 'catdir|c=s' => \$catdir, + 'region|r=s' => \$region, + 'dbname|d=s' => \$dbname,# Database name + 'workdir|w=s' => \$workdir, # Working directory for output files + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dvo_id --catdir --region", + -exitval => 3) unless + defined $dvo_id and + defined $catdir and + defined $region; + +# Look for programs we need +my $missing_tools; +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $uniphot = can_run('uniphot') or (warn "Can't find uniphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $caltool = can_run('caltool') or (warn "Can't find caltool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# select the primary filters from DVO query? +my (@filters) = `photcodeList -average`; + +# parse the region (RAs,RAe:DECs,DECe) : item = +/-NNN.NNNN +my @coords = split (":", $region); +my ($RAs, $RAe) = split (",", $coords[0]); +my ($DECs, $DECe) = split (",", $coords[1]); + +# Run addstar -resort +{ + my $command = "$addstar -resort"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RESORT", $status, $dbname); + } +} + +# Run relphot (filter) for each filter +{ + foreach my $filter (@filters) { + my $command = "$relphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); + } + } +} + +# Run uniphot (filter) for each filter +# XXX skip this one? run less frequently? +if (0) { + foreach my $filter (@filters) { + my $command = "$uniphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "UNIPHOT", $status, $dbname); + } + } +} + +{ + my $command = "$relastro -objects"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + } +} + +{ + my $command = "$relastro -images"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.IMAGES", $status, $dbname); + } +} + +my $command = "$caltool -addcalrun"; +$command .= " -dvo_id $dvo_id"; +$command .= " -region $region"; +$command .= " -last_step RELASTRO.IMAGES"; +$command .= " -status SUCCESS"; +$command .= " -dbname $dbname" if defined $dbname; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $dvo_id = shift; + my $region = shift; + my $last_step = shift; + my $status = shift; + my $dbname = shift; + + carp($msg); + if (defined $dvo_id && defined $region && defined $last_step && defined $status and not $no_update) { + my $command = "$caltool -addcalrun"; + $command .= " -dvo_id $dvo_id"; + $command .= " -region $region"; + $command .= " -last_step $last_step"; + $command .= " -status $status"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/flatcorr_proc.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/flatcorr_proc.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/flatcorr_proc.pl (revision 22293) @@ -0,0 +1,214 @@ +#!/usr/bin/env perl + +## USAGE:flatcorr_proc.pl --dbname --corr_id + +## this script does the following steps: + +# extract the details of the flatcorr run: dvodb, filter, camera, etc? + +# relphot -D CATDIR $dvodb -grid (outgrid.fits) (filter) -region 0 360 -90 90 (other parameters?) + +# dvoMakeCorr -file outgrid.fits -ref ref.fits outcorr + +# dettool -register -det_type FLATCORR -filelevel (level) -workdir -inst, etc +# foreach $imfile () +# dettool -register_imfile -uri, etc, etc + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($corr_id, $dvodb, $region, $filter, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'corr_id|i=s' => \$corr_id, + 'dvodb|c=s' => \$dvodb, + 'region|r=s' => \$region, + 'filter|f=s' => \$filter, + 'dbname|d=s' => \$dbname,# Database name + 'workdir|w=s' => \$workdir, # Working directory for output files + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --corr_id --dvodb --region --filter", + -exitval => 3) unless + defined $corr_id and + defined $dvodb and + defined $region and + defined $filter; + +# Look for programs we need +my $missing_tools; +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $dvoMakeCorr = can_run('dvoMakeCorr') or (warn "Can't find dvoMakeCorr" and $missing_tools = 1); +my $detselect = can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $flatcorr = can_run('flatcorr') or (warn "Can't find flatcorr" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +my($outgrid, $outcorr); + +$outgrid = "$workdir/grid.$corr_id.fits"; +$outcorr = "$workdir/corr.$corr_id.fits"; + +# parse the region (RAs,RAe:DECs,DECe) : item = +/-NNN.NNNN +my @coords = split (":", $region); +my ($RAs, $RAe) = split (",", $coords[0]); +my ($DECs, $DECe) = split (",", $coords[1]); + +# Run relphot (filter) for the specified region (need to clarify the options like imfreeze) +{ + my $command = "$relphot $filter"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + $command .= "-grid $outgrid"; + $command .= "-imfreeze?"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform relphot -grid on region $region: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); + } +} + +# use one of the input raw images as a reference image +# XXX I'm not sure how this works: do I run this once per imfile for a given exp? +my ($reffile) +{ + my $command = "$flatcoor -flatcorrimfile -limit 1"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform flatcorr -flatcorrimfile: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $dvo_id, $region, "RELPHOT", $status, $dbname, $PS_EXIT_PROG_ERROR);); + + my $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $dvo_id, $region, "RELPHOT", $status, $dbname, $PS_EXIT_PROG_ERROR);); + + my $imfile = $imfiles->[0]; + $reffile = $imfile->{uri}; +} + +{ + my $command = "$dvoMakeCorr -file $outgrid -ref $reffile $outcorr"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform dvoMakeCorr: $error_code", $dvo_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + } +} + +{ + my $command = "$dettool -register"; + $command .= "-det_type FLATCORR"; + $command .= "-file_level $fileLevel"; + $command .= "-workdir $workdir"; + $command .= "-inst $inst"; + $command .= " -dbname $dbname" if defined $dbname; + ## XXX what else do we need, and where do we get it? + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.IMAGES", $status, $dbname); + } +} + +my $command = "$flatcorr -done"; +$command .= " -corr_id $corr_id"; +$command .= " -stats UNKNOWN"; +$command .= " -dbname $dbname" if defined $dbname; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $dvo_id = shift; + my $region = shift; + my $last_step = shift; + my $status = shift; + my $dbname = shift; + + carp($msg); + if (defined $dvo_id && defined $region && defined $last_step && defined $status and not $no_update) { + my $command = "$caltool -addcalrun"; + $command .= " -dvo_id $dvo_id"; + $command .= " -region $region"; + $command .= " -last_step $last_step"; + $command .= " -status $status"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/gpc_seeing.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/gpc_seeing.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/gpc_seeing.pl (revision 22293) @@ -0,0 +1,28 @@ +#!/usr/bin/env perl +# basic ISP transmission analysis: + +if (@ARGV != 1) { die "USAGE: gpc_seeing.pl (input.fits)\n"; } +$input = $ARGV[0]; + +# for input file /path/foo.fits, use /path/foo for output + +@words = split ('\.', $input); +if (@words > 1) { pop @words; } +$output = join (".", @words); + +# use constant RECIPE => 'PPIMAGE_OBDSFRA'; # Recipe to use +$RECIPE_PPIMAGE = 'PPIMAGE_OP'; +$RECIPE_PSPHOT = 'PSPHOT.SEEING'; + +# recommend only processing to PSFMODEL +vsystem ("ppImage -file $input $output -recipe PPIMAGE $RECIPE_PPIMAGE -recipe PSPHOT $RECIPE_PSPHOT"); +if ($status) { die "failure running ppImage\n"; } + +# XXX otis can read the output psf model, or we can supply a program to interpret the model + +sub vsystem { + print STDERR "@_\n"; + my $status = system ("@_"); + $status; +} + Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_darkstats.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_darkstats.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_darkstats.pl (revision 22293) @@ -0,0 +1,198 @@ +#!/usr/bin/env perl + +# use warnings; +# use strict; +use Carp; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use IO::Handle; + +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); # tools to parse the IPP configuration information + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, $det_id, $camera); + +GetOptions('dbname=s' => \$dbname, + 'det_id=s' => \$det_id, + 'camera|c=s' => \$camera, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "USAGE: ipp_darkstats.pl --dbname (name) --det_id (id) --camera (name)", + -exitval => 3, + ) unless defined $dbname and defined $det_id and defined $camera; + +$ipprc->define_camera($camera); + +### Get list of dark imfile results + +# define the dettool command +my $command = "dettool -processedimfile -select_state stop"; # Command to run +$command .= " -det_id $det_id"; +$command .= " -dbname $dbname" if defined $dbname; + +# run the dettool command and catch the output +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", $error_code); +} + +# parse the output into a list +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_PROG_ERROR); +my $list = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $PS_EXIT_PROG_ERROR); + +my @bg_data; +my @bg_stdev_data; +my @bg_name; +my @bg_exptime; +my %components; + +print STDERR "extracted the data from the database\n"; + +# we now have a list of imfiles; we need to extract the background for each cell +# from the stats files for each imfile +foreach my $item (@$list) { + my $path_base = $item->{path_base}; + my $class_id = $item->{class_id}; + my $exp_time = $item->{exp_time}; + + my $rootName = $ipprc->file_resolve ($path_base); + my $statsName = "$rootName.$class_id.stats"; + + # print STDERR "rootName: $rootName : $exp_time\n"; + print STDERR "statsName: $statsName : $exp_time\n"; + + my $statsFile; + open $statsFile, $statsName; + my @contents = <$statsFile>; + close ($statsFile); + + # print STDERR "contents: @contents\n"; + + my $parser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $statsList = $parser->parse(join "", @contents) or &my_die("Unable to parse metadata for imfile stats", $PS_EXIT_SYS_ERROR); + + &parse_stats_table ($exp_time, $class_id, $statsList); +} + +print STDERR "parsed the stats from the data files\n"; + +for (my $i = 0; $i < @bg_data; $i++) { + $nameX = "$bg_name[$i].exp"; + $nameY = "$bg_name[$i].bg"; + push @{$nameX}, $bg_exptime[$i]; + push @{$nameY}, $bg_data[$i]; +} + +if (-e "output.dat") { unlink "output.dat"; } + +print STDERR "dumping stats\n"; +open (MANA, "|mana --norc"); +MANA->autoflush; + +foreach my $component (@components) { + $nameX = "$component.exp"; + $nameY = "$component.bg"; + + print MANA "delete X Y\n"; + + open (DATA, ">$component.dat"); + for (my $i = 0; $i < @{$nameX}; $i++) { + print DATA "${$nameX}[$i] ${$nameY}[$i]\n"; + } + close (DATA); + + print MANA "data $component.dat\n"; + print MANA "read X 1 Y 2\n"; + print MANA "fit X Y 2 -clip 3 3\n"; + print MANA "output output.dat\n"; + print MANA "echo $component METADATA\n"; + print MANA "echo \" NORDER_X S32 2 \"\n"; + print MANA "echo \" VAL_X00 F64 \$C0\"\n"; + print MANA "echo \" VAL_X01 F64 \$C1\"\n"; + print MANA "echo \" VAL_X02 F64 \$C2\"\n"; + print MANA "echo \" NELEMENTS S32 3 \"\n"; + print MANA "echo END\n"; + print MANA "echo\n"; + print MANA "output stdout\n"; + + print MANA "applyfit X Yf\n"; + print MANA "lim X Y\n"; + print MANA "clear\n"; + print MANA "box\n"; + print MANA "plot -x 2 -pt 2 -sz 1.0 -c black X Y\n"; + print MANA "plot -x 2 -pt 7 -sz 1.0 -c red X Yf\n"; + + print STDERR "hit return to continue\n"; + $answer = ; +} + +close (MANA); + +exit 0; + +sub parse_stats_table +{ + my ($exp_time, $tag, $md) = @_; + + # descend through the fpa + foreach my $entry (@$md) { + # print STDERR "name: $entry->{name}, class: $entry->{class}\n"; + # recurse on nested metadata + if ($entry->{class} eq 'metadata') { + my $newtag = $tag . "_" . $entry->{name}; + &parse_stats_table ($exp_time, $newtag, $entry->{value}); + } + + if ($entry->{name} =~ /^(SAMPLE|ROBUST|FITTED|CLIPPED)/) { + # It's a statistic of some sort + if ($entry->{name} =~ /_STDEV$/) { + push @bg_stdev_data, $entry->{value}; + } else { + push @bg_name, $tag; + push @bg_data, $entry->{value}; + push @bg_exptime, $exp_time; + # print STDERR "$tag $exp_time $entry->{value}\n"; + } + if (!$componentsHash{$tag}) { + push @components, $tag; + $componentsHash{$tag} = 1; + } + next; + } + } + return 1; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exit_code = shift; # Exit code to add + + carp($msg); + exit $exit_code; +} + +# - get the exp_time as well from dettool +# - build an array of bg & exptime for each cell +# - fit the trend (in mana? pslib functions?) +# - write the polynomial for each cell Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_datapath.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_datapath.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_datapath.pl (revision 22293) @@ -0,0 +1,15 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use PS::IPP::Config; +my $ipprc = PS::IPP::Config->new(); + +die "No filename specified.\n" if scalar @ARGV != 1; + +print $ipprc->file_resolve(shift @ARGV) . "\n"; + +1; + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_detrend_combine.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_detrend_combine.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_detrend_combine.pl (revision 22293) @@ -0,0 +1,228 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Data::Dumper; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use PS::IPP::Metadata::Stats; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $RECIPE_PPSTATS = 'CHIPSTATS'; # Recipe to use with ppStats + +# Parse command-line arguments +my ($det_type, $filelevel, $inst, $telescope, $filter, + $det_id1, $iter1, $det_id2, $iter2, $operation, $mask, + $workdir, $dbname, $no_update); +GetOptions( + 'det_type=s' => \$det_type, # Detrend type for new detrend + 'filelevel=s' => \$filelevel, # File level for new detrend + 'inst=s' => \$inst, # Instrument for new detrend + 'telescope=s' => \$telescope, # Telescope for new detrend + 'filter=s' => \$filter, # Filter name for new detrend + 'det_id1=s' => \$det_id1, # Detrend id for detrend 1 + 'iteration1=s' => \$iter1, # Iteration for detrend 1 + 'det_id2=s' => \$det_id2, # Detrend id for detrend 2 + 'iteration2=s' => \$iter2, # Iteration for detrend 2 + 'operation=s' => \$operation, # Operation to perform on files + 'mask' => \$mask, # Operation is on a mask + 'workdir=s' => \$workdir, # Working directory for output files + 'dbname=s' => \$dbname, # Database name + 'no-update' => \$no_update, # Don't update the database + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options --det_type --filelevel --inst --telescope --det_id1 --iteration1 --det_id2 --iteration2 --workdir", + -exitval => 3, + ) + unless defined $det_type + and defined $filelevel + and defined $inst + and defined $telescope + and defined $det_id1 + and defined $iter1 + and defined $det_id2 + and defined $iter2 + and defined $operation + and defined $workdir; + +$ipprc->define_camera($inst); + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; + +# Look for programs we need +my $missing_tools; +my $detselect = can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppArith = can_run('ppArith') or (warn "Can't find ppArith" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get the list of inputs +my $files1 = filelist($det_id1, $iter1); # Hash of input files for detrend 1 +my $files2 = filelist($det_id2, $iter2); # Hash of input files for detrend 2 +die("File lists for detrends have differing lengths") unless scalar keys %$files1 == scalar keys %$files2; + +my ($det_id, $iter); # Detrend identifier for the new detrend +unless ($no_update) { + my $command = "$dettool -register_detrend -det_type $det_type -filelevel $filelevel -workdir $workdir " . + "-inst $inst -telescope $telescope"; # Command to run + $command .= " -filter $filter" if defined $filter; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run dettool -register_detrend: $error_code"); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or die("Unable to parse metadata config doc\n"); + my $md = parse_md_list($metadata) or die("Unable to parse metadata list\n"); + + $det_id = $$md[0]->{det_id}; + $iter = $$md[0]->{iteration}; + + die("Unable to get det_id and iteration for new detrend.\n") unless defined $det_id and defined $iter; +} else { + $det_id = 'DUMMY_DET_ID'; + $iter = 'DUMMY_ITER'; +} + +my $outRoot = caturi($workdir, "$inst.$det_id.$iter"); # Output root name +my $filerule = (defined $mask ? "PPARITH.OUTPUT.MASK" : "PPARITH.OUTPUT.IMAGE"); # File rule for ppArith + +foreach my $class_id ( keys %$files1 ) { + my $md1 = $$files1{$class_id}; + my $md2 = $$files2{$class_id}; + die("Class_id=$class_id not defined for det_id=$det_id2") unless defined $md2; + + my $uri1 = $$md1[0]->{uri}; + my $uri2 = $$md2[0]->{uri}; + + die("Unable to find input file $uri1\n") unless $ipprc->file_exists($uri1); + die("Unable to find input file $uri2\n") unless $ipprc->file_exists($uri2); + + my $outName = $ipprc->filename($filerule, $outRoot, $class_id); + my $outStats = $outRoot . '.stats'; + + my $command = "$ppArith -file1 $uri1 -op \'$operation\' -file2 $uri2 $outRoot"; # Command to run + $command .= " -stats $outStats -recipe PPSTATS $RECIPE_PPSTATS"; + $command .= ' -mask' if defined $mask; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run ppArith: $error_code"); + } + + die("Unable to find ppArith product: $outName\n") unless $ipprc->file_exists($outName); + die("Unable to find ppArith product: $outStats\n") unless $ipprc->file_exists($outStats); + + # Get the statistics on the processed image + my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + { + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outStats) or die("Can't open stats file $outStats: $!"); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + my $metadata = $mdcParser->parse(join "", @contents) or die("Unable to parse metadata config doc"); + + unless ($stats->parse($metadata)) { + &my_die("Failure extracting metadata from the statistics output file.\n"); + } + } + + # Register the imfile + unless ($no_update) { + my $command = "$dettool -register_detrend_imfile -det_id $det_id "; # Command to run + $command .= " -class_id $class_id -uri $outName -path_base $outRoot"; + $command .= $stats->cmdflags(); + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run dettool -register_detrend_imfile: $error_code"); + } + } +} + + +### Pau. + + +# Get a list of files for the given detrend +sub filelist +{ + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + + my $command = "$detselect -select -det_id $det_id -iteration $iter"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run detselect: $error_code"); + } + + # Because of the length, need to split into individual metadatas --- it parses SO much quicker! + my %files; + + my $md = $mdcParser->parse( join( "", @$stdout_buf ) ); # Parsed metadata + my $list = parse_md_list( $md ); + + foreach my $item ( @$list ) { + my $class_id = $item->{class_id}; + die("Multiple definitions of class_id=$class_id found for det_id=$det_id, iteration=$iter\n") if + defined $files{$class_id}; + $files{$class_id} = parse_md_list($md); + } + + return \%files; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_filename.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_filename.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_filename.pl (revision 22293) @@ -0,0 +1,48 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# use these to check the apache environment +#print "$ENV{'PATH'}\n"; +#print "$ENV{'PERL5LIB'}\n"; + +use PS::IPP::Config; +my $ipprc = PS::IPP::Config->new(); + +my ($filerule, $class_id, $basename, $camera); + +GetOptions('filerule=s' => \$filerule, + 'class_id=s' => \$class_id, + 'basename=s' => \$basename, + 'camera|c=s' => \$camera, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --filerule --class_id --basename --camera", + -exitval => 3, +) unless defined $basename + and defined $filerule + and defined $class_id + and defined $camera; + +$ipprc->define_camera($camera); + +#print "$filerule\n"; +#print "$basename\n"; +#print "$camera\n"; +#print "$class_id\n"; + +my $filename = $ipprc->filename($filerule, $basename, $class_id); +#print "$filename\n"; + +my $realname = $ipprc->file_resolve( $filename ); +print "$realname\n"; + +1; + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_inject_fileset.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_inject_fileset.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_inject_fileset.pl (revision 22293) @@ -0,0 +1,148 @@ +#!/usr/bin/env perl + +# this program injects a set of files the db, assuming the list of +# files all belong to a single exposure the user supplies a temporary +# telescope and camera name. these are used for informational +# purposes only until the registration step can determine the true +# telescope and camera name from the image headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $dbname, $no_op, $help); +GetOptions('camera|i=s' => \$camera, # user-supplied camera name + 'telescope|t=s' => \$telescope, # user-supplied telescope name + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'label=s' => \$label, # set chip label + 'dbname|d=s' => \$dbname, # Database name + 'no-op' => \$no_op, # pretend but don't actually inject + 'help' => \$help # give help listing + ) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many files into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--label label] [--dbname dbname] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +# XXX why are these required? +pod2usage( -msg => "Required options: --telescope (name) --camera (name)", + -exitval => 3) unless @ARGV > 0; + +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# if workdir is not defined, assign the current path +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +if (! $telescope) { + $telescope = "UNKNOWN"; +} + +if (! $camera) { + $camera = "UNKNOWN"; +} + +# use the first file name as the exp_name (strip off .fits) +my $num = 0; +my $exp_name; +foreach my $file ( @ARGV ) { + # check for file existence + if (! -e $file) { die "file $file not found\n"; } + if (! $exp_name) { + # strip off the extension + my ( $vol, $path, $name ) = File::Spec->splitpath( $file ); + ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts)(|.gz)/; + print "exp_name : $exp_name.\n"; + } + $num ++; +} + +print "$num files in fileset.\n"; + +## inject the exposure + +# the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values +my $command_exp = "$pxinject -newExp"; +$command_exp .= " -tmp_exp_name $exp_name"; +$command_exp .= " -tmp_inst $camera"; +$command_exp .= " -tmp_telescope $telescope"; +$command_exp .= " -workdir $workdir"; +$command_exp .= " -reduction $reduction" if defined $reduction; +$command_exp .= " -dvo_db $dvo_db" if defined $dvo_db; +$command_exp .= " -tess_id $tess_id" if defined $tess_id; +$command_exp .= " -end_stage $end_stage" if defined $end_stage; +$command_exp .= " -label $label" if defined $label; +$command_exp .= " -dbname $dbname" if defined $dbname; + +my $exp_id = 0; +unless ($no_op) { + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + $exp_id = $line[2]; # The exposure tag +} else { + print "skipping command: $command_exp\n"; +} + +# now inject the imfiles one at a time +for (my $i = 0; $i < @ARGV; $i++) { + + my $file = $ARGV[$i]; + + my $absfile = File::Spec->rel2abs( $file ); + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id file.$i"; + $command_imfile .= " -uri $relfile"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } else { + print "skipping command: $command_imfile\n"; + } +} + +# the class_id used here is temporary : register replaces it with the true class_id +my $command_update = "$pxinject -updatenewExp"; +$command_update .= " -exp_id $exp_id"; +$command_update .= " -state run"; +$command_update .= " -dbname $dbname" if defined ($dbname); + +unless ($no_op) { + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; +} else { + print "skipping command: $command_update\n"; +} + +exit 0; + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_maskscript.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_maskscript.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_maskscript.pl (revision 22293) @@ -0,0 +1,199 @@ +#!/usr/bin/env perl + +# use warnings; +# use strict; +use Carp; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use IO::Handle; + +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); # tools to parse the IPP configuration information + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, $det_id, $camera); + +GetOptions('dbname=s' => \$dbname, + 'det_id=s' => \$det_id, + 'camera|c=s' => \$camera, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "USAGE: ipp_maskscript.pl --dbname (name) --det_id (id) --iter (iteration) --camera (name)", + -exitval => 3, + ) unless defined $dbname and defined $det_id and defined $camera; + +# I could determine the camera from a query for the detrun +$ipprc->define_camera($camera); + +### Get list of dark imfile results + +# define the dettool command +my $command = "dettool -processedimfile -select_state stop"; # Command to run +$command .= " -det_id $det_id"; +$command .= " -dbname $dbname" if defined $dbname; + +# run the dettool command and catch the output +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", $error_code); +} + +# parse the output into a list +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_PROG_ERROR); +my $list = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $PS_EXIT_PROG_ERROR); + +my @bg_data; +my @bg_stdev_data; +my @bg_name; +my @bg_exptime; +my %components; + +print STDERR "extracted the data from the database\n"; + +# we now have a list of imfiles; we need to extract the background for each cell +# from the stats files for each imfile +foreach my $item (@$list) { + my $path_base = $item->{path_base}; + my $class_id = $item->{class_id}; + my $exp_time = $item->{exp_time}; + + my $rootName = $ipprc->file_resolve ($path_base); + my $statsName = "$rootName.$class_id.stats"; + + # print STDERR "rootName: $rootName : $exp_time\n"; + print STDERR "statsName: $statsName : $exp_time\n"; + + my $statsFile; + open $statsFile, $statsName; + my @contents = <$statsFile>; + close ($statsFile); + + # print STDERR "contents: @contents\n"; + + my $parser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $statsList = $parser->parse(join "", @contents) or &my_die("Unable to parse metadata for imfile stats", $PS_EXIT_SYS_ERROR); + + &parse_stats_table ($exp_time, $class_id, $statsList); +} + +print STDERR "parsed the stats from the data files\n"; + +for (my $i = 0; $i < @bg_data; $i++) { + $nameX = "$bg_name[$i].exp"; + $nameY = "$bg_name[$i].bg"; + push @{$nameX}, $bg_exptime[$i]; + push @{$nameY}, $bg_data[$i]; +} + +if (-e "output.dat") { unlink "output.dat"; } + +print STDERR "dumping stats\n"; +open (MANA, "|mana --norc"); +MANA->autoflush; + +foreach my $component (@components) { + $nameX = "$component.exp"; + $nameY = "$component.bg"; + + print MANA "delete X Y\n"; + + open (DATA, ">$component.dat"); + for (my $i = 0; $i < @{$nameX}; $i++) { + print DATA "${$nameX}[$i] ${$nameY}[$i]\n"; + } + close (DATA); + + print MANA "data $component.dat\n"; + print MANA "read X 1 Y 2\n"; + print MANA "fit X Y 2 -clip 3 3\n"; + print MANA "output output.dat\n"; + print MANA "echo $component METADATA\n"; + print MANA "echo \" NORDER_X S32 2 \"\n"; + print MANA "echo \" VAL_X00 F64 \$C0\"\n"; + print MANA "echo \" VAL_X01 F64 \$C1\"\n"; + print MANA "echo \" VAL_X02 F64 \$C2\"\n"; + print MANA "echo \" NELEMENTS S32 3 \"\n"; + print MANA "echo END\n"; + print MANA "echo\n"; + print MANA "output stdout\n"; + + print MANA "applyfit X Yf\n"; + print MANA "lim X Y\n"; + print MANA "clear\n"; + print MANA "box\n"; + print MANA "plot -x 2 -pt 2 -sz 1.0 -c black X Y\n"; + print MANA "plot -x 2 -pt 7 -sz 1.0 -c red X Yf\n"; + + print STDERR "hit return to continue\n"; + $answer = ; +} + +close (MANA); + +exit 0; + +sub parse_stats_table +{ + my ($exp_time, $tag, $md) = @_; + + # descend through the fpa + foreach my $entry (@$md) { + # print STDERR "name: $entry->{name}, class: $entry->{class}\n"; + # recurse on nested metadata + if ($entry->{class} eq 'metadata') { + my $newtag = $tag . "_" . $entry->{name}; + &parse_stats_table ($exp_time, $newtag, $entry->{value}); + } + + if ($entry->{name} =~ /^(SAMPLE|ROBUST|FITTED|CLIPPED)/) { + # It's a statistic of some sort + if ($entry->{name} =~ /_STDEV$/) { + push @bg_stdev_data, $entry->{value}; + } else { + push @bg_name, $tag; + push @bg_data, $entry->{value}; + push @bg_exptime, $exp_time; + # print STDERR "$tag $exp_time $entry->{value}\n"; + } + if (!$componentsHash{$tag}) { + push @components, $tag; + $componentsHash{$tag} = 1; + } + next; + } + } + return 1; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exit_code = shift; # Exit code to add + + carp($msg); + exit $exit_code; +} + +# - get the exp_time as well from dettool +# - build an array of bg & exptime for each cell +# - fit the trend (in mana? pslib functions?) +# - write the polynomial for each cell Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_camera.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_camera.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_camera.pl (revision 22293) @@ -0,0 +1,86 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $workdir_default, # Default working directory + $verbose, # Verbose operations? + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'workdir=s' => \$workdir_default, + 'verbose' => \$verbose, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +$workdir_default = `pwd` unless defined $workdir_default; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser +my $ipprc = PS::IPP::Config->new; # IPP Configuration + +# Look for programs we need +my $missing_tools; +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $camera_exp = can_run('camera_exp.pl') or (warn "Can't find camera_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Camera exposure processing +my $list; +{ + my $command = "$camtool -pendingexp -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get camera exposure list: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from camtool.\n"; +} + +foreach my $item (@$list) { + my $cam_id = $item->{cam_id}; + my $exp_tag = $item->{exp_tag}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL"); + my $outroot = caturi( $workdir, $exp_tag, "$exp_tag.cm.$cam_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$camera_exp --cam_id $cam_id --exp_tag $exp_tag --camera $camera --dbname $dbname"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do camera processing on $cam_id: $error_code\n" if not $success; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_chip.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_chip.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_chip.pl (revision 22293) @@ -0,0 +1,105 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $workdir_default, # Default working directory + $verbose, # Verbose operations? + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'workdir=s' => \$workdir_default, + 'verbose' => \$verbose, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( -msg => "Required options: --dbname --workdir", + -exitval => 3, + ) unless + defined $dbname; + +$workdir_default = `pwd` unless defined $workdir_default; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser +my $ipprc = PS::IPP::Config->new; # IPP Configuration + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $chip = can_run('chip_imfile.pl') or (warn "Can't find chip_imfile.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Imfile processing +my @whole; # The whole list for processing +{ + my $command = "$chiptool -pendingimfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 2 imfile list: $error_code\n" if not $success; + @whole = split /\n/, join( '', @$stdout_buf ); +} + +my @single = (); + +while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from chiptool.\n"; + + foreach my $item (@$list) { + my $chip_id = $item->{chip_id}; + my $exp_id = $item->{exp_id}; + my $exp_tag = $item->{exp_tag}; + my $camera = $item->{camera}; + my $class_id = $item->{class_id}; + my $uri = $item->{uri}; + my $reduction = $item->{reduction}; + my $workdir = $item->{workdir}; + $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL"); + + my $outroot = caturi( $workdir, $exp_tag, "$exp_tag.ch.$chip_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$chip --chip_id $chip_id --exp_id $exp_id --exp_tag $exp_tag --class_id $class_id --uri $uri --dbname $dbname --camera $camera --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 2 processing on $chip_id $class_id: $error_code\n" if not $success; + } + + @single = (); + + } +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_detrend.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_detrend.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_detrend.pl (revision 22293) @@ -0,0 +1,366 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use Getopt::Long; +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config qw( caturi ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config; +use Data::Dumper; + +my ($dbname, # Database name to use + $workdir_global # Global working directory + ); +GetOptions( + 'dbname=s' => \$dbname, + 'workdir=s' => \$workdir_global, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $detrend_process_imfile = can_run('detrend_process_imfile.pl') or (warn "Can't find detrend_process_imfile.pl" and $missing_tools = 1); +my $detrend_process_exp = can_run('detrend_process_exp.pl') or (warn "Can't find detrend_process_exp.pl" and $missing_tools = 1); +my $detrend_stack = can_run('detrend_stack.pl') or (warn "Can't find detrend_stack.pl" and $missing_tools = 1); +my $detrend_norm_calc = can_run('detrend_norm_calc.pl') or (warn "Can't find detrend_norm_calc.pl" and $missing_tools = 1); +my $detrend_norm_apply = can_run('detrend_norm_apply.pl') or (warn "Can't find detrend_norm_apply.pl" and $missing_tools = 1); +my $detrend_norm_exp = can_run('detrend_norm_exp.pl') or (warn "Can't find detrend_norm_exp.pl" and $missing_tools = 1); +my $detrend_resid = can_run('detrend_resid.pl') or (warn "Can't find detrend_resid.pl" and $missing_tools = 1); +my $detrend_reject_imfile = can_run('detrend_reject_imfile.pl') or (warn "Can't find detrend_reject_imfile.pl" and $missing_tools = 1); +my $detrend_reject_exp = can_run('detrend_reject_exp.pl') or (warn "Can't find detrend_reject_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Process raw imfiles +{ + my $command = "$dettool -toprocessedimfile -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get detrend raw list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $det_type = $item->{det_type}; + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $class_id = $item->{class_id}; + my $uri = $item->{uri}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + my $reduction = $item->{reduction}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", $exp_tag, "$exp_tag.detproc.$det_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_process_imfile --det_id $det_id --exp_tag $exp_tag --exp_id $exp_id --class_id $class_id --det_type $det_type --input_uri $uri --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do raw imfile processing on $exp_tag $class_id: $error_code\n" if not $success; + } + + @single = (); + + } + } +} + +# Process raw exposures +{ + my $command = "$dettool -toprocessedexp -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get detrend raw list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", $exp_tag, "$exp_tag.detproc.$det_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_process_exp --det_id $det_id --det_type $det_type --exp_tag $exp_tag --exp_id $exp_id --camera $camera --dbname $dbname --outroot $outroot"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do raw exposure processing on $det_id $exp_tag: $error_code\n" if not $success; + } +} + +# Stack +{ + my $command = "$dettool -tostacked -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get stack list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $class_id = $item->{class_id}; + my $det_type = $item->{det_type}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + my $reduction = $item->{reduction}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + + my $command = "$detrend_stack --det_id $det_id --iteration $iteration --class_id $class_id --det_type $det_type --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to stack detrend $det_id $iteration $class_id: $error_code\n" if not $success; + } +} + +# Calculate normalisation +{ + my $command = "$dettool -tonormalizedstat -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get normalise calculation list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.normstat.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_norm_calc --det_id $det_id --iteration $iteration --det_type $det_type --dbname $dbname --outroot $outroot"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to calculate normalisation for $det_id $iteration: $error_code\n" if not $success; + } +} + +# Apply normalisation +{ + my $command = "$dettool -tonormalize -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get normalisation list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $class_id = $item->{class_id}; + my $value = $item->{norm}; + my $uri = $item->{uri}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.norm.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_norm_apply --det_id $det_id --iteration $iteration --class_id $class_id --value $value --input_uri $uri --camera $camera --det_type $det_type --dbname $dbname --outroot $outroot"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to apply normalisation for $det_id $iteration $class_id: $error_code\n" if not $success; + } +} + +# Examine normalised exposure +{ + my $command = "$dettool -tonormalizedexp -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get normalised exposures list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $class_id = $item->{class_id}; + my $value = $item->{norm}; + my $uri = $item->{uri}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.normexp.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_norm_exp --det_id $det_id --iteration $iteration --camera $camera --det_type $det_type --dbname $dbname --outroot $outroot"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to examine normalised exposure for $det_id $iteration: $error_code\n" if not $success; + } +} + +# Get residuals +{ + my $command = "$dettool -toresidimfile -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get residual processing list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $class_id = $item->{class_id}; + my $det_type = $item->{det_type}; + my $detrend = $item->{det_uri}; + my $uri = $item->{uri}; + my $mode = $item->{mode}; + my $workdir = $item->{workdir}; + my $reduction = $item->{reduction}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", $exp_tag, "$exp_tag.detresid.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_resid --det_id $det_id --iteration $iteration --exp_tag $exp_tag --exp_id $exp_id --class_id $class_id --det_type $det_type --camera $camera --input_uri $uri --mode $mode --dbname $dbname --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --detrend $detrend" if defined $detrend; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do residual processing on $exp_tag $class_id: $error_code\n" if not $success; + } + + @single = (); + + } + } +} + +# Reject based on imfiles +{ + my $command = "$dettool -toresidexp -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get residual imfile list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", $exp_tag, "$exp_tag.detresid.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_reject_imfile --det_id $det_id --iteration $iteration --exp_tag $exp_tag --exp_id $exp_id --det_type $det_type --camera $camera --dbname $dbname --outroot $outroot"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do imfile rejection on $exp_tag $det_id $iteration: $error_code\n" if not $success; + } +} + +# Reject based on exposures +{ + my $command = "$dettool -todetrunsummary -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get residual exposure list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.$det_id.$iteration.detreject" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_reject_exp --det_id $det_id --iteration $iteration --det_type $det_type --camera $camera --dbname $dbname --outroot $outroot"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do exposure rejection on $det_id $iteration: $error_code\n" if not $success; + } +} + + +__END__ + + Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_diff.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_diff.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_diff.pl (revision 22293) @@ -0,0 +1,102 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, # Database name to use + $verbose, # Verbose operations? + $workdir_global, # Global working directory + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'workdir' => \$workdir_global, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $difftool = can_run('difftool') or + (warn "Can't find difftool" and $missing_tools = 1); +my $diff_skycell = can_run('diff_skycell.pl') or + (warn "Can't find diff_skycell.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Image differencing +{ + my $list; + my $command = "$difftool -todiffskyfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get list of diffs: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from difftool.\n"; + + foreach my $item (@$list) { + my $diff_id = $item->{diff_id}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "tess_$tess_id", $skycell_id, "$tess_id.$skycell_id.dif.$diff_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$diff_skycell --diff_id $diff_id --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do difference for $diff_id: $error_code\n" if not $success; + } +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_inject.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_inject.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_inject.pl (revision 22293) @@ -0,0 +1,142 @@ +#!/usr/bin/env perl + +# this program injects a list of single-file exposures into the db, +# taking the filename (without .fits) as the exp_tag. the user +# supplies a temporary telescope and camera name. these are used +# for informational purposes only until the registration step can +# determine the true telescope and camera name from the image +# headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $dbname, $no_op, $help); +GetOptions('camera|i=s' => \$camera, # user-supplied camera name + 'telescope|t=s' => \$telescope, # user-supplied telescope name + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'label=s' => \$label, # set chip label + 'dbname|d=s' => \$dbname, # Database name + 'no-op' => \$no_op, # pretend but don't actually inject + 'help' => \$help # give help listing +) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many files into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--label label] [--dbname dbname] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( -msg => "Required options: --telescope (name) --camera (name)", + -exitval => 3) unless + defined $telescope and + defined $camera; + +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# if workdir is not defined, assign the current path +# XXX we need to handle relative paths for workdir (not allowed) +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +my $num = 0; +foreach my $file ( @ARGV ) { + # check for file existence + if (! -e $file) { die "file $file not found\n"; } + my $absfile = File::Spec->rel2abs( $file ); + inject($absfile, $workdir, $dbname, $telescope, $camera); + $num ++; +} + +print "$num files injected.\n"; + +sub inject +{ + my $absfile = shift; # absolute path for this file + my $workdir = shift; # absolute path for output directory + my $dbname = shift; # IPP database to use + my $telescope = shift; # user-specified telescope + my $camera = shift; # user-specified camera + + # XXX provide an option for an alternative extension + my ( $vol, $path, $name ) = File::Spec->splitpath( $absfile ); + my ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts)(|.gz)/; + + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + # the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values + my $command_exp = "$pxinject -newExp"; + $command_exp .= " -tmp_exp_name $exp_name"; + $command_exp .= " -tmp_inst $camera"; + $command_exp .= " -tmp_telescope $telescope"; + $command_exp .= " -workdir $workdir"; + $command_exp .= " -reduction $reduction" if defined $reduction; + $command_exp .= " -dvodb $dvo_db" if defined $dvo_db; + $command_exp .= " -tess_id $tess_id" if defined $tess_id; + $command_exp .= " -end_stage $end_stage" if defined $end_stage; + $command_exp .= " -label $label" if defined $label; + $command_exp .= " -dbname $dbname" if defined $dbname; + + my $exp_id = 0; + unless ($no_op) { + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + $exp_id = $line[2]; # The exposure tag + } else { + print "skipping command: $command_exp\n"; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id fpa"; + $command_imfile .= " -uri $relfile"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } else { + print "skipping command: $command_imfile\n"; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_update = "$pxinject -updatenewExp"; + $command_update .= " -exp_id $exp_id"; + $command_update .= " -state run"; + $command_update .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; + } else { + print "skipping command: $command_update\n"; + } + + return 1; +} + + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_inject_mosaic.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_inject_mosaic.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_inject_mosaic.pl (revision 22293) @@ -0,0 +1,166 @@ +#!/usr/bin/env perl + +# this program injects a set of multi-file exposures into the db. the +# program takes a list of base exposure names and injects all files +# associated with the exposure. It constructs the expected filenames +# from the exposure tag and rules for the camera. This program is for +# the test only since it requires too much information at the inject +# stage. use 'ipp_serial_inject.pl' for single-file images and +# 'ipp_serial_inject_split.pl' for multiple file images in split +# format + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($camera, # Camera used + $telescope, # Telescope used + $dbname, # Database name + $workdir, # Working directory + $path, # Path to data + ); +GetOptions( + 'camera|c=s' => \$camera, + 'telescope|t=s' => \$telescope, + 'workdir=s' => \$workdir, + 'path=s' => \$path, + 'dbname=s' => \$dbname, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --camera --telescope --workdir --path --dbname", + -exitval => 3, + ) unless defined $camera + and defined $telescope + and defined $workdir + and defined $path + and defined $dbname; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +# Look for programs we need +my $missing_tools; +my $pxinject = can_run('pxinject') or (warn "Can't find pxinject" and $missing_tools = 1); + +if (scalar @ARGV == 0) { + die "No exposures provided.\n"; +} + +# Inject new data into the database +my @classes; # Names of the classes +my @files; # What to add to the filename for each class +my $imfiles; +my $add_dir; # Add directory name to get file name? +if ($camera eq "MEGACAM") { + for (my $i = 0; $i < 36; $i++) { + push @classes, sprintf("ccd%02d", $i); + push @files, sprintf(".ccd%02d", $i); + } +} elsif ($camera eq "MCSHORT") { + @classes = ( 'ccd12', 'ccd13', 'ccd14', 'ccd21', 'ccd22', 'ccd23' ); + @files = ( '.ccd12', '.ccd13', '.ccd14', '.ccd21', '.ccd22', '.ccd23' ); +} elsif ($camera eq "CTIO_MOSAIC2") { + @classes = (); + @files = (); +} elsif ($camera eq "TC3") { + @classes = ( 'CCID58-1-06b2', 'CCID45-1-14A', 'CCID45-1-11A', 'CCID45-1-22A', + 'CCID45-1-04C', 'CCID45-1-13A', 'CCID45-1-05A', 'CCID45-1-19A' ); + @files = ( '00', '01', '10', '11', '20', '21', '30', '31' ); + $add_dir = 1; +} elsif ($camera eq "SIMMOSAIC") { + @classes = ( 'Chip00', 'Chip01', 'Chip10', 'Chip11' ); + @files = ( '.Chip00', '.Chip01', '.Chip10', '.Chip11' ); +} elsif ($camera eq "SIMTEST") { + @classes = (); + @files = (); +} elsif ($camera eq "GPC1") { + for (my $i = 0; $i < 8; $i++) { + for (my $j = 0; $j < 8; $j++) { + if (($i == 0 or $i == 7) and ($j == 0 or $j == 7)) { + # Excluding corner chips + next; + } + push @classes, "XY$i$j"; + push @files, "$i$j"; + } + } + $add_dir = 1; +} else { + die "Unrecognised camera name: $camera.\n"; +} + +foreach my $exp_name ( @ARGV ) { + my $command = "$pxinject -newExp -tmp_exp_name $exp_name -tmp_inst $camera -tmp_telescope $telescope -workdir $workdir"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to inject $exp_name: $error_code\n" if not $success; + + my @line = split(/\s+/, $$stdout_buf[0]); # The output line, containing the exposure tag + my $exp_id = $line[2]; # The exposure tag + for (my $i = 0; $i < scalar @classes; $i++) { + my $class_id = $classes[$i]; + my $file_id = $files[$i]; + my $filename = $exp_name . $file_id . '.fits'; + $filename = caturi( $exp_name, $filename ) if defined $add_dir; + $filename = caturi( $path, $filename ); + + die "Unable to find file $filename" unless -f $ipprc->file_resolve( $filename ); + + $filename = $ipprc->convert_filename_relative( $filename ); + my $command = "$pxinject -newImfile -exp_id $exp_id -tmp_class_id $class_id -uri $filename"; # Command to run + $command .= " -dbname $dbname" if defined ($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to inject $exp_name $class_id: $error_code\n" if not $success; + } + + if (scalar @classes == 0) { + my $filename = $exp_name . '.fits'; + $filename = caturi( $exp_name, $filename ) if defined $add_dir; + $filename = caturi( $path, $filename ); + + die "Unable to find file $filename" unless -f $ipprc->file_resolve( $filename ); + + $filename = $ipprc->convert_filename_relative( $filename ); + my $command = "$pxinject -newImfile -exp_id $exp_id -tmp_class_id fpa -uri $filename"; # Command to run + $command .= " -dbname $dbname" if defined ($dbname); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code\n" if not $success; + } + + # Update the exposure to run + { + my $command = "$pxinject -updatenewExp -exp_id $exp_id -state run"; # Command to run + $command .= " -dbname $dbname" if defined ($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to activate $exp_name: $error_code\n" if not $success; + } + +} + +END { + my $status = $?; +system("sync") == 0 + or die "failed to execute sync: $!" ; +$? = $status; +} + + +__END__ + + Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_inject_split.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_inject_split.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_inject_split.pl (revision 22293) @@ -0,0 +1,138 @@ +#!/usr/bin/env perl + +# this program injects a list of multi-file exposures into the db. +# the program takes a list of directory names (dir), and injects all +# dir/*.fits. It takes the directory name as the exp_tag. the user +# supplies a temporary telescope and instrument name. these are used +# for informational purposes only until the registration step can +# determine the true telescope and instrument name from the image +# headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $dbname, $help); +GetOptions('camera|c=s' => \$camera, # Camera used + 'telescope|t=s' => \$telescope, # Telescope used + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'dbname|d=s' => \$dbname, # Database name + 'help' => \$help # give help listing +) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many image exposures (directories) into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--dbname name] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( + -msg => "Required options: --camera --telescope", + -exitval => 3) unless + defined $telescope and + defined $camera; + +# Look for programs we need +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# if workdir is not defined, assign the current path +# we need to handle relative paths for workdir (not allowed) +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +my $num = 0; +foreach my $filedir ( @ARGV ) { + # check for filedir existence + if (! -e $filedir) { die "file dir $filedir not found\n"; } + my $absfiledir = File::Spec->rel2abs( $filedir ); + inject($absfiledir, $workdir, $dbname, $telescope, $camera); + $num ++; +} + +sub inject +{ + my $absfiledir = shift; # absolute path for this file + my $workdir = shift; # absolute path for output directory + my $dbname = shift; # IPP database to use + my $telescope = shift; # user-specified telescope + my $camera = shift; # user-specified camera + + # XXX provide an option for an alternative extension + my @files = <$absfiledir/*.fits>; + my $Nfiles = scalar @files; + + print "absfiledir: $absfiledir\n"; + print "Nfiles: $Nfiles\n"; + + # the absfiledir is of the form path://PATH/data/foo/expname + my ( $vol, $path, $exp_name ) = File::Spec->splitpath( $absfiledir ); + + # the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values + my $command_exp = "$pxinject -newExp"; + $command_exp .= " -tmp_exp_name $exp_name"; + $command_exp .= " -tmp_inst $camera"; + $command_exp .= " -tmp_telescope $telescope"; + $command_exp .= " -workdir $workdir"; + $command_exp .= " -reduction $reduction" if defined $reduction; + $command_exp .= " -dvodb $dvo_db" if defined $dvo_db; + $command_exp .= " -tess_id $tess_id" if defined $tess_id; + $command_exp .= " -end_stage $end_stage" if defined $end_stage; + $command_exp .= " -dbname $dbname" if defined $dbname; + + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + my $exp_id = $line[2]; # The exposure tag + + foreach my $absfile (@files) { + + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + my ( $tmpvol, $tmppath, $filename ) = File::Spec->splitpath( $absfile ); + my ( $class_name ) = $filename =~ /(.*)\.fits/; + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id $class_name"; + $command_imfile .= " -uri $relfile"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_update = "$pxinject -updatenewExp"; + $command_update .= " -exp_id $exp_id"; + $command_update .= " -state run"; + $command_update .= " -dbname $dbname" if defined ($dbname); + + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; + + return 1; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_register.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_register.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_register.pl (revision 22293) @@ -0,0 +1,122 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + ); +GetOptions( + 'dbname|d=s' => \$dbname, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $register_imfile = can_run('register_imfile.pl') or (warn "Can't find register_imfile.pl" and $missing_tools = 1); +my $register_exp = can_run('register_exp.pl') or (warn "Can't find register_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Phase 0 imfile processing +{ + my $command = "$regtool -pendingimfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 0 imfile list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from regtool.\n"; + + foreach my $item (@$list) { + my $exp_id = $item->{exp_id}; + my $exp_name = $item->{tmp_exp_name}; + my $class_id = $item->{tmp_class_id}; + my $workdir = $item->{workdir}; + my $uri = $item->{uri}; + + my $command = "$register_imfile --exp_id $exp_id --tmp_class_id $class_id --tmp_exp_name $exp_name --uri $uri --dbname $dbname"; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 0 imfile processing on $exp_name $class_id: $error_code\n" if not $success; + } + + + @single = (); + + } + } +} + +# Phase 0 exposure processing +{ + my $command = "$regtool -pendingexp -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 0 exposure list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from regtool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{tmp_exp_name} . '.' . $item->{exp_id}; + my $exp_id = $item->{exp_id}; + my $workdir = $item->{workdir}; + + my $command = "$register_exp --exp_tag $exp_tag --exp_id $exp_id --dbname $dbname"; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 0 exposure processing on $exp_tag: $error_code\n" if not $success; + } + + @single = (); + + } + } +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_stack.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_stack.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_stack.pl (revision 22293) @@ -0,0 +1,107 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, # Database name to use + $verbose, # Verbose operations? + $workdir_global, # Global working directory + $no_op, # No operations? + $no_update, # No updating? + $save_temps, # Save temporary files? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'workdir' => \$workdir_global, + 'no-op' => \$no_op, + 'no-update' => \$no_update, + 'save-temps' => \$save_temps, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $stacktool = can_run('stacktool') or + (warn "Can't find difftool" and $missing_tools = 1); +my $stack_skycell = can_run('stack_skycell.pl') or + (warn "Can't find stack_skycell.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Image stacking +{ + my $list; + my $command = "$stacktool -tosum -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get list of stacks: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from stacktool.\n"; + + foreach my $item (@$list) { + my $stack_id = $item->{stack_id}; + my $workdir = $item->{workdir}; + my $tess_id = $item->{tess_id}; + my $skycell_id = $item->{skycell_id}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "tess_$tess_id", $skycell_id, "$tess_id.$skycell_id.stk.$stack_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$stack_skycell --stack_id $stack_id --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + $command .= " --save-temps" if defined $save_temps; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); +### die "Unable to do stack for $stack_id: $error_code\n" if not $success; + } +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_warp.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_warp.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_serial_warp.pl (revision 22293) @@ -0,0 +1,137 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, # Database name to use + $verbose, # Verbose operations? + $workdir_global, # Global working directory + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'workdir' => \$workdir_global, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or + (warn "Can't find warptool" and $missing_tools = 1); +my $warp_skycell = can_run('warp_skycell.pl') or + (warn "Can't find warp_skycell.pl" and $missing_tools = 1); +my $warp_overlap = can_run('warp_overlap.pl') or + (warn "Can't find warp_overlap.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Calculate overlaps +{ + my $list; + my $command = "$warptool -tooverlap -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get warps for which to calculate overlaps: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from warptool.\n"; + + foreach my $item (@$list) { + my $warp_id = $item->{warp_id}; + my $cam_id = $item->{cam_id}; + my $workdir = $item->{workdir}; + my $camera = $item->{camera}; + my $tess_id = $item->{tess_id}; + + my $command = "$warp_overlap --warp_id $warp_id --camera $camera --tess_id $tess_id --dbname $dbname"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get warp overlaps on $warp_id: $error_code\n" if not $success; + } +} + + +# Warping proper +{ + my $list; + my $command = "$warptool -towarped -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get warps for warping: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from warptool.\n"; + + foreach my $item (@$list) { + my $warp_id = $item->{warp_id}; + my $skycell_id = $item->{skycell_id}; + my $tess_id = $item->{tess_id}; + my $cam_id = $item->{cam_id}; + my $workdir = $item->{workdir}; + my $camera = $item->{camera}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "tess_$tess_id", $skycell_id, "$tess_id.$skycell_id.wrp.$warp_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$warp_skycell --warp_id $warp_id --skycell_id $skycell_id --tess_id $tess_id --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do warp processing on $warp_id,$skycell_id: $error_code\n" if not $success; + } +} + + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_simulation_data.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_simulation_data.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipp_simulation_data.pl (revision 22293) @@ -0,0 +1,220 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use Math::Trig; +use File::Spec; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($name, # Base name for output images + $camera, # Name of camera to use + $telescope, # Telescope name + $dbname, # Database name + $path, # Path to data + $workdir, # Working directory for data + $no_cal, # Don't produce calibration files + $no_update # Don't update the database + ); + +GetOptions( + 'name=s' => \$name, + 'camera=s' => \$camera, + 'telescope=s' => \$telescope, + 'dbname=s' => \$dbname, + 'path=s' => \$path, + 'workdir=s' => \$workdir, + 'no-cal' => \$no_cal, + 'no-update' => \$no_update, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --name --path --camera --telescope --dbname", + -exitval => 3, + ) unless + defined $name and + defined $path and + defined $camera and + defined $telescope and + defined $dbname; + +$workdir = $path if not defined $workdir; + +# Look for programs we need +my $missing_tools; +my $ppSim = can_run('ppSim') + or (warn "Can't find ppSim" and $missing_tools = 1); +my $inject = can_run('ipp_serial_inject_mosaic.pl') + or (warn "Can't find ipp_serial_inject_mosaic.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Number of bias images +use constant BIAS => 20; +# Dark exposure times +use constant DARK => [ 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 300, 300, 300, 300, 300, 300, 300, 300, 300, 300 ]; +# Flat-field image characteristics +use constant FLAT => [ + { + filter => 'r', + exptime => [ 0.1, 0.1, 0.1, 0.1, 0.1, 0.5, 1, 2, 5, 10, 20, 20, 20, 20, 20 ], + }, + { + filter => 'i', + exptime => [ 20, 20, 20, 20, 20 ], + } + ]; +# Object image characteristics +use constant OBJECT => [ + { + filter => 'r', + exptime => [ 5, 10, 10, 10, 10, 10, 240 ], + seeing => [ 0.8, 0.4, 0.6, 0.8, 1.2, 1.5, 0.7 ], + ra => 150.119167, + dec => 2.205833, + pa => 0, + zp => 25.15, + sky => 20.86, + dither => 40, + }, + { + filter => 'i', + exptime => [ 5, 30, 30, 30, 30, 30, 240 ], + seeing => [ 0.8, 0.4, 0.6, 0.8, 1.2, 1.5, 0.7 ], + ra => 150.119167, + dec => 2.205833, + pa => 0, + zp => 25.00, + sky => 20.15, + dither => 40, + }, + ]; +use constant SCALE => 0.2; # Plate scale + +############################################################################################################# +### Now do the work +############################################################################################################# + +my $counter = 0; + +# Generate bias images +for (my $i = 0; $i < BIAS; $i++) { + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + unless ($no_cal) { + run( command => "$ppSim -camera $camera -type BIAS $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "--workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } +} + +# Generate dark images +foreach my $exptime ( @{DARK()} ) { + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + unless ($no_cal) { + run ( command => "$ppSim -camera $camera -type DARK -exptime $exptime $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "--workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } +} + +# Generate flat images +foreach my $set ( @{FLAT()} ) { + my $filter = $set->{filter}; # Name of filter + foreach my $exptime ( @{$set->{exptime}} ) { + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + unless ($no_cal) { + run( command => "$ppSim -camera $camera -type FLAT -filter $filter -exptime $exptime $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "--workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } + } +} + +# Generate object images +foreach my $set ( @{OBJECT()} ) { + my $filter = $set->{filter}; # Name of filter + my $ra0 = $set->{ra}; # Base Right Ascension (deg) + my $dec0 = $set->{dec}; # Base Declination (deg) + my $pa = $set->{pa}; # Position angle (deg) + my $zp = $set->{zp}; # Zero point + my $scale = SCALE(); # Plate scale (arcsec/pix) + my $sky = 10**( -0.4 * ( $set->{sky} - $zp ) ) * $scale**2; # Sky background (counts/s) + my $dither = $set->{dither} / 3600; # Dither size (deg) + + for (my $i = 0; $i < scalar @{$set->{exptime}}; $i++) { + my $exptime = ${$set->{exptime}}[$i]; # Exposure time + my $seeing = ${$set->{seeing}}[$i]; # Seeing (pix) + my $ra = $ra0 + (2*rand() - 1) * $dither * cos(deg2rad($dec0)); # RA with dither + my $dec = $dec0 + (2*rand() - 1) * $dither; # Dec with dither + + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + run( command => "$ppSim -camera $camera -type OBJECT -filter $filter -exptime $exptime " . + "-skyrate $sky -ra $ra -dec $dec -pa $pa -scale $scale -zp $zp -seeing $seeing $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "--workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } +} + + +### Pau. + + +# Generate a filename from the base name and counter +sub filename +{ + my $base = shift; # Base name + my $num = shift; # Number + my $workdir = shift; # Working directory + my $name = sprintf("$base%04d", $num); + $num++; + return ( $name, $num ); +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipprc.txt =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipprc.txt (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/ipprc.txt (revision 22293) @@ -0,0 +1,16 @@ + +these scripts use the following library functions to work with the IPP Config system: + +$ipprc->convert_filename_absolute( $tess_dir ); +$ipprc->convert_filename_relative( $absfile ); +$ipprc->define_camera($camera); +$ipprc->extname_rule("CMF.HEAD", $class_id); # MEF psastro output +$ipprc->file_create_append( $logName ); +$ipprc->file_exists( $skyfile ); +$ipprc->file_prepare( "$exp_tag/$exp_tag.detproc.$det_id", $workdir, $input_uri ); +$ipprc->file_resolve ($path_base); +$ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); +$ipprc->outroot_prepare($outroot); +$ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use +$ipprc->rejection( $name, $det_type, $filter ); +$ipprc->tessellation_catdir( $tess_id ); # Tessellation catdir for DVO Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/isp_trans.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/isp_trans.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/isp_trans.pl (revision 22293) @@ -0,0 +1,30 @@ +#!/usr/bin/env perl +# basic ISP transmission analysis: + +if (@ARGV != 1) { die "USAGE: isp_trans.pl (input.fits)\n"; } +$input = $ARGV[0]; + +# for input file /path/foo.fits, use /path/foo for output + +@words = split ('\.', $input); +if (@words > 1) { pop @words; } +$output = join (".", @words); + +# use constant RECIPE => 'PPIMAGE_OBDSFRA'; # Recipe to use +$RECIPE_PPIMAGE = 'PPIMAGE_OA'; # Recipe to use (switch to OBDSFRA when detrend images are ready) +$RECIPE_PSPHOT = 'PSPHOT.SUMMIT'; +$CALDIR = '/data/alala.0/ipp/ippRefs/catdir.synth.bright'; # source of photometric calibration data +$IMTABLE = 'images.dat'; # source of photometric calibration data + +vsystem ("ppImage -file $input $output -recipe PPIMAGE $RECIPE_PPIMAGE -recipe PSPHOT $RECIPE_PSPHOT"); +if ($status) { die "failure running ppImage\n"; } + +vsystem ("addstar -incal -image -D CAMERA isp -D IMAGE_TABLE $IMTABLE -D CATDIR $CALDIR $output.smf"); +if ($status) { die "failure getting calibration from addstar\n"; } + +sub vsystem { + print STDERR "@_\n"; + my $status = system ("@_"); + $status; +} + Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/magic_tree.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/magic_tree.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/magic_tree.pl (revision 22293) @@ -0,0 +1,337 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use Astro::FITS::CFITSIO qw( :constants ); +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +use Math::Trig; +use File::Temp qw( tempfile ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use constant MAX_FIELDS => 4; # Maximum number of fields to be in a node + +# Parse the command-line arguments +my ($magic_id, $tess_id, $camera, $ra0, $dec0, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op); +GetOptions( + 'magic_id=s' => \$magic_id, # Magic identifier + 'tess_id=s' => \$tess_id, # Tessellation identifier + 'camera=s' => \$camera, # Camera name + 'ra=f' => \$ra0, # Boresight right ascension, radians + 'dec=f' => \$dec0, # Boresight declination, radians + 'dbname=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'save-temps' => \$save_temps, # Save temporary files? + 'verbose' => \$verbose, # Print stuff? + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --magic_id --camera --outroot", + -exitval => 3) unless + defined $magic_id and + defined $tess_id and + defined $ra0 and + defined $dec0 and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + + +# Look for programs we need +my $missing_tools; +my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +### Get a list of skycells +my @skycells; # List of skycells +{ + my $command = "$magictool -inputskyfile -magic_id $magic_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magictool -inputfile: $error_code", $magic_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR); + + my $inputs = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $magic_id, $PS_EXIT_PROG_ERROR); + + foreach my $input ( @$inputs ) { + push @skycells, $input->{skycell_id}; + } +} + +### For each skycell, project centre of skycell onto tangent plane of boresight +my @fields; +foreach my $skycell_id ( @skycells ) { + my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id ); + $ipprc->skycell_file($tess_id, $skycell_id, $skyFile, $verbose) or &my_die("Unable to generate skycells $skycell_id", $magic_id, $PS_EXIT_PROG_ERROR); + my $skyfileResolved = $ipprc->file_resolve( $skyfile ); + my ($header, $status) = Astro::FITS::CFITSIO::fits_read_header( $skyfileResolved ); + &my_die("Unable to read skycell header: $status", $magic_id, $PS_EXIT_SYS_ERROR) if $status; + + # Get the useful header keywords + my $naxis1 = $$header{'NAXIS1'} or &my_die("Can't find NAXIS1", $magic_id, $PS_EXIT_SYS_ERROR); + my $naxis2 = $$header{'NAXIS2'} or &my_die("Can't find NAXIS2", $magic_id, $PS_EXIT_SYS_ERROR); + my $ctype1 = $$header{'CTYPE1'} or &my_die("Can't find CTYPE1", $magic_id, $PS_EXIT_SYS_ERROR); + my $ctype2 = $$header{'CTYPE2'} or &my_die("Can't find CTYPE2", $magic_id, $PS_EXIT_SYS_ERROR); + my $cdelt1 = $$header{'CDELT1'} or &my_die("Can't find CDELT1", $magic_id, $PS_EXIT_SYS_ERROR); + my $cdelt2 = $$header{'CDELT2'} or &my_die("Can't find CDELT2", $magic_id, $PS_EXIT_SYS_ERROR); + my $crval1 = $$header{'CRVAL1'} or &my_die("Can't find CRVAL1", $magic_id, $PS_EXIT_SYS_ERROR); + my $crval2 = $$header{'CRVAL2'} or &my_die("Can't find CRVAL2", $magic_id, $PS_EXIT_SYS_ERROR); + my $crpix1 = $$header{'CRPIX1'} or &my_die("Can't find CRPIX1", $magic_id, $PS_EXIT_SYS_ERROR); + my $crpix2 = $$header{'CRPIX2'} or &my_die("Can't find CRPIX2", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc11 = $$header{'PC001001'} or &my_die("Can't find PC001001", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc12 = $$header{'PC001002'} or &my_die("Can't find PC001002", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc21 = $$header{'PC002001'} or &my_die("Can't find PC002001", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc22 = $$header{'PC002002'} or &my_die("Can't find PC002002", $magic_id, $PS_EXIT_SYS_ERROR); + my $crota1 = $$header{'CROTA1'}; + my $crota2 = $$header{'CROTA2'}; + + &my_die("Unexpected projection: $ctype1 and $ctype2.", $magic_id, $PS_EXIT_SYS_ERROR) unless + $ctype1 =~ /^\'RA---TAN\s*\'$/ and $ctype2 =~ /^\'DEC--TAN\s*\'$/; + &my_die("Can't determine size of skycell ($naxis1,$naxis2)", $magic_id, $PS_EXIT_SYS_ERROR) unless + $naxis1 > 0 and $naxis2 > 0; + &my_die("Can't determine scale of skycell ($cdelt1,$cdelt2)", $magic_id, $PS_EXIT_SYS_ERROR) if + not defined $cdelt1 or $cdelt1 == 0 or not defined $cdelt2 or $cdelt2 == 0; + &my_die("We don't know how to handle rotations ($crota1,$crota2)", $magic_id, $PS_EXIT_SYS_ERROR) + if defined $crota1 or defined $crota2; + + # Relative coordinates of centre of the field + my $x = $naxis1 - $crpix1; + my $y = $naxis2 - $crpix2; + + # Coordinates on tangent plane + my $xi = $pc11 * ($x) + $pc12 * ($y); + my $eta = $pc21 * ($x) + $pc22 * ($y); + $xi *= $cdelt1; + $eta *= $cdelt2; + + # Coordinates on rotated celestial sphere + my $phi = atan2($eta,$xi) + pi/2; + my $theta = atan(180 / pi / sqrt($xi**2 + $eta**2)); + + # Coordinates on celestial sphere + $crval1 = deg2rad($crval1); + $crval2 = deg2rad($crval2); + my $ra = $crval1 + atan2(cos($theta) * sin($phi), + sin($theta) * cos($crval2) + cos($theta) * sin($crval2) * cos($phi)); + my $dec = asin(sin($theta) * sin($crval2) - cos($theta) * cos($crval2) * cos($phi)); + + # Rotate to boresight + my $phi_new = atan2(cos($dec) * sin($ra - $ra0), + sin($dec) * cos($dec0) + cos($dec) * sin($dec0) * cos($ra - $ra0)); + my $theta_new = asin(sin($dec) * sin($dec0) - cos($dec) * cos($dec0) * cos($ra - $ra0)); + + # Project + my $rad = 180 / pi * cot($theta_new); + my $xi_new = $rad * sin($phi); + my $eta_new = - $rad * cos($phi); + + my $field = { id => $skycell_id, + xi => $xi_new, + eta => $eta_new, + }; + + push @fields, $field; +} + +### Subdivide list of positions into kd-tree +my $root = { # Root node of tree + contents => \@fields, # Contents of node + position => 'root', # Position in tree + children => {}, # Children of node +}; +my @tasks = ( $root ); +while (scalar @tasks > 0) { + my $node = shift @tasks; + divide_node($node, \@tasks); +} + +### Format tree for magictool +my $mdcTree = print_node($root); # The tree in MDC format +my ($treeFile, $treeName) = tempfile( "magictree.${magic_id}.XXXX", UNLINK => !$save_temps ); +print $treeFile, $mdcTree; +close $treeFile; + +### Input tree into database +{ + my $command = "$magictool -inputtree"; + $command .= " -magic_id $magic_id"; + $command .= " -dep_file $treeName"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform magictool -inputtree: $error_code"); + exit($error_code); + } + } else { + print "Skipping command: $command\n"; + } +} + +### Pau. + +sub my_die +{ + my $msg = shift; # Warning message on die + my $magic_id = shift; # Magic identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $magic_id and not $no_update) { + my $command = "$magictool -inputtree"; + $command .= " -magic_id $magic_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system($command); + } + exit $exit_code; +} + +# Divide a list into two, returning the lower and upper parts +sub divide_list +{ + my $list = shift; # List to divide + my $index = shift; # Name of index for sorting + + my @sorted = sort { $$a{$index} <=> $$b{$index} } @$list; # Sorted list + my $median = int(scalar @sorted / 2); # Median point of list + my @upper = splice(@sorted, $median); # Upper part of the sorted list + + return (\@sorted, \@upper); +} + +# Create a new node, add it to the parent, and add it to the task list if required +sub new_node +{ + my $parent = shift; # The parent node + my $contents = shift; # Contents of the new node + my $position = shift; # Position description + my $tasks = shift; # Tasks to do + + my $node = { + contents => $contents, + position => $parent->{position} . '_' . $position, + children => {}, + }; + + $parent->{children}->{$position} = $node; + + push @$tasks, $node if scalar @$contents > 4; + + return $node; +} + +# Divide a node +sub divide_node +{ + my $node = shift; # Node to divide + my $tasks = shift; # Tasks to do + + my $position = $node->{position}; + + my $contents = $node->{contents} or die "Can't find contents of node."; # Contents of node + + my ($lower, $upper) = divide_list($contents, 'xi'); + + if (scalar @$lower > 4) { + my ($ll, $lr) = divide_list($lower, 'eta'); + new_node($node, $ll, 'll', $tasks); + new_node($node, $lr, 'lr', $tasks); + } else { + new_node($node, $lower, 'L', $tasks); + } + + if (scalar @$upper > 4) { + my ($ul, $ur) = divide_list($upper, 'eta'); + new_node($node, $ul, 'ul', $tasks); + new_node($node, $ur, 'ur', $tasks); + } else { + new_node($node, $upper, 'U', $tasks); + } + + $node->{contents} = undef; + + return $node; +} + +# Print the contents of a node +sub print_node +{ + my $node = shift; # Node to print + + my $position = $node->{position}; # Position of node + + my $output = "$position\t\tMULTI\n"; # Output text + + if (defined $node->{contents}) { + foreach my $field ( @{$node->{contents}} ) { + my $skycell_id = $field->{id}; # Skycell name + $output .= "$position\t\tSTR\t$skycell_id\n"; + $output .= "$skycell_id\t\tSTR\tNULL\t\# $field->{xi},$field->{eta}\n"; + } + } else { + foreach my $div ( keys %{$node->{children}} ) { + $output .= "$position\t\tSTR\t${position}_$div\n"; + $output .= print_node($node->{children}->{$div}); + } + } + + return $output; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/mdc2list.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/mdc2list.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/mdc2list.pl (revision 22293) @@ -0,0 +1,77 @@ +#!/usr/bin/env perl + +# Program to convert a PS metadata config syntax (usually from one of +# the IPP tools) into a flat tab-delimited list suitable for parsing +# using "split". + +# Copyright (C) 2006 Joshua Hoblitt, Paul A. Price. + +use strict; +use warnings; + +use PS::IPP::Metadata::Config; # Supplies the metadata config parser. + +die "Program to convert a PS metadata config file (usually from one of the IPP\n" . + "tools) into a flat tab-delimited list suitable for parsing using \'split\'.\n\n" . + "Usage: $0 [INFILE [OUTFILE]]\n" if ((join "", @ARGV) =~ /--help/ or scalar @ARGV > 2); + +my $inFile; # Input file +my $outFile; # Output file +if (scalar @ARGV >= 1) { + my $inName = shift @ARGV; + open $inFile, $inName or die "Can't open $inName: $!\n"; + if (scalar @ARGV == 1) { + my $outName = shift @ARGV; + open $outFile, ">", $outName or die "Can't open $outName: $!\n"; + } else { + $outFile = *STDOUT; + } +} else { + $inFile = *STDIN; + $outFile = *STDOUT; +} + +my @input = <$inFile>; # Contents of the metadata config file +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $md = $mdcParser->parse(join "", @input) + or die "unable to parse metadata config doc"; +my $hashes = mds2hashes($md); # An array of hashes +foreach my $pending (@$hashes) { + foreach my $key (keys %$pending) { + print $outFile ( $pending->{$key} . "\t"); + } + print $outFile "\n"; +} + +### Pau. + + +# Given an array of MDs, return an array of hashes +sub mds2hashes +{ + my $mds = shift; # Reference to the metadatas + my @array; # The array of hashes, to be returned + foreach my $md (@$mds) { + my $values = md2hash($md->{value}); + push @array, $values; + } + return \@array; +} + +# Convert the metadata to a hash; in effect, strips out the comment, type and class fields. +sub md2hash +{ + my $values = shift; # Reference to the metadata + my %hash; # Hash, to be returned + foreach my $data (@$values) { + $hash{$data->{name}} = $data->{value}; + } + return \%hash; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/register_exp.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/register_exp.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/register_exp.pl (revision 22293) @@ -0,0 +1,224 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Cache::File; +use Storable qw( freeze thaw ); +use File::Basename qw( basename ); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my ($cache, $exp_id, $exp_tag, $dbname, $verbose, $no_update, $no_op); +GetOptions( + 'caches' => \$cache, + 'exp_id|e=s' => \$exp_id, + 'exp_tag|t=s' => \$exp_tag, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --exp_tag", + -exitval => 3) unless + defined $exp_id and + defined $exp_tag; + +# add -detrend UNLESS type is one of SCIENCE listed below (eg, OBJECT) +my @SCIENCE = ( "object", "science" ); # Observation types to NOT mark as detrend +my $DETREND_FLAG = "-end_stage reg"; # Flag to use to mark detrend exposure + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ # register imfile + # label STATISTIC CHIPTOOL FLAG + { name => "exp_name", type => "constant", flag => "-exp_name", dtype => "string" }, # File level + { name => "telescope", type => "constant", flag => "-telescope", dtype => "string" }, # File level + { name => "camera", type => "constant", flag => "-inst", dtype => "string" }, # File level + { name => "filelevel", type => "constant", flag => "-filelevel", dtype => "string" }, # File level + { name => "object", type => "constant", flag => "-object", dtype => "string" }, + { name => "exp_type", type => "constant", flag => "-exp_type", dtype => "string" }, # File level + { name => "filter", type => "constant", flag => "-filter", dtype => "string" }, # File level + { name => "comment", type => "constant", flag => "-comment", dtype => "string" }, # ObsComment + { name => "dateobs", type => "constant", flag => "-dateobs", dtype => "string" }, # File level + { name => "ccd_temp", type => "mean", flag => "-ccd_temp", dtype => "float" }, # CCD temperature + { name => "exp_time", type => "mean", flag => "-exp_time", dtype => "float" }, # Exposure time + { name => "sat_pixel_frac", type => "mean", flag => "-sat_pixel_frac", dtype => "float" }, # Fraction of saturated pixels + { name => "airmass", type => "mean", flag => "-airmass", dtype => "float" }, # Airmass + { name => "ra", type => "mean", flag => "-ra", dtype => "float" }, # Right ascension + { name => "decl", type => "mean", flag => "-decl", dtype => "float" }, # Declination + { name => "posang", type => "mean", flag => "-posang", dtype => "float" }, # Position angle + { name => "alt", type => "mean", flag => "-alt", dtype => "float" }, # Altitude + { name => "az", type => "mean", flag => "-az", dtype => "float" }, # Azimuth + { name => "m1_x", type => "constant", flag => "-m1_x", dtype => "float" }, # M1X + { name => "m1_y", type => "constant", flag => "-m1_y", dtype => "float" }, # M1Y + { name => "m1_z", type => "constant", flag => "-m1_z", dtype => "float" }, # M1Z + { name => "m1_tip", type => "constant", flag => "-m1_tip", dtype => "float" }, # M1TIP + { name => "m1_tilt", type => "constant", flag => "-m1_tilt", dtype => "float" }, # M1TILT + { name => "m2_x", type => "constant", flag => "-m2_x", dtype => "float" }, # M2X + { name => "m2_y", type => "constant", flag => "-m2_y", dtype => "float" }, # M2Y + { name => "m2_z", type => "constant", flag => "-m2_z", dtype => "float" }, # M2Z + { name => "m2_tip", type => "constant", flag => "-m2_tip", dtype => "float" }, # M2TIP + { name => "m2_tilt", type => "constant", flag => "-m2_tilt", dtype => "float" }, # M2TILT + { name => "env_temperature", type => "constant", flag => "-env_temperature", dtype => "float" }, # external temp + { name => "env_humidity", type => "constant", flag => "-env_humidity", dtype => "float" }, # external humidity + { name => "env_wind_speed", type => "constant", flag => "-env_wind_speed", dtype => "float" }, # external wind speed + { name => "env_wind_dir", type => "constant", flag => "-env_wind_dir", dtype => "float" }, # external wind direction + + { name => "-teltemp_m1", type => "constant", flag => "-teltemp_m1", dtype => "float" }, # Primary mirror temps (C) + { name => "-teltemp_m1cell", type => "constant", flag => "-teltemp_m1cell", dtype => "float" }, # Primary mirror support temps (C) + { name => "-teltemp_m2", type => "constant", flag => "-teltemp_m2", dtype => "float" }, # Secondary mirror temps (C + { name => "-teltemp_spider", type => "constant", flag => "-teltemp_spider", dtype => "float" }, # Spider temperatures (C) + { name => "-teltemp_truss", type => "constant", flag => "-teltemp_truss", dtype => "float" }, # Mid truss temperatures (C + { name => "-teltemp_extra", type => "constant", flag => "-teltemp_extra", dtype => "float" }, # Miscellaneous temperatures (C) + + { name => "pon_time", type => "mean", flag => "-pon_time", dtype => "float" }, # time since last power on + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, # background + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, # Azimuth + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, # Azimuth + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for commands we need +my $missing_tools; +my $regtool = can_run('regtool') + or (warn "can't find regtool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# setup cache interface +my $c = Cache::File->new( + cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)), + default_expires => '7200 sec', +); + +# Get the list of imfiles & their stats +{ + my $command = "$regtool -processedimfile -exp_id $exp_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -processedimfile on exposure id $exp_id: $error_code"); + exit ($error_code); + } + + # Parse the output + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + unless ($metadata) { + &my_die ("Unable to parse metadata config doc", $exp_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die ("Unable to find all values", $exp_id, $PS_EXIT_CONFIG_ERROR); + } +} + +# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type +if (uc($stats->value_for_flag ("-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-inst")) eq "NULL") { &my_die ("inst not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-exp_type")) eq "NULL") { &my_die ("exp_type not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-exp_name")) eq "NULL") { &my_die ("exp_name not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } + +my $command = "$regtool -addprocessedexp"; +$command .= " -exp_id $exp_id"; +$command .= " -exp_tag $exp_tag"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +my $exp_type = $stats->value_for_flag ("-exp_type"); + +# Add the detrend flag, if needed +{ + my $object = 0; # Is it an object exposure? + foreach my $scienceType (@SCIENCE) { + if (lc($exp_type) =~ /$scienceType/) { + $object = 1; + last; + } + } + $command .= " $DETREND_FLAG" unless $object; +} + +# Output results to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to run regtool -addprocessedexp for $exp_id: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +### Pau. + +sub cache_run +{ + my %p = @_; + + my $cmd_output = $c->get($p{command}) if $cache; + if (defined $cmd_output) { + return @{thaw $cmd_output}; + } else { + my @output = run(%p); + $c->set($p{command}, freeze \@output) if $cache; + return @output; + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; + my $exit_code = shift; + + carp($msg); + if (defined $exp_id and not $no_update) { + my $command = "$regtool -addprocessedexp -exp_id $exp_id -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/register_imfile.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/register_imfile.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/register_imfile.pl (revision 22293) @@ -0,0 +1,246 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Cache::File; +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $dbname, $verbose, $no_update, $no_op); +GetOptions( + 'caches' => \$cache, + 'exp_id|e=s' => \$exp_id, + 'tmp_class_id|i=s' => \$tmp_class_id, + 'tmp_exp_name|n=s' => \$tmp_exp_name, + 'uri|u=s' => \$uri, + 'dbname|d=s' => \$dbname,# Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --tmp_class_id --tmp_exp_name --uri", + -exitval => 3) unless + defined $exp_id and + defined $tmp_class_id and + defined $tmp_exp_name and + defined $uri; + +my $RECIPE = "REGISTER"; # Recipe to use for ppStats + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "FILE.LEVEL", type => "constant", flag => "-filelevel", dtype => "string" }, # File level + { name => "CLASS.ID", type => "constant", flag => "-class_id", dtype => "string" }, # Real Class ID + { name => "FPA.OBJECT", type => "constant", flag => "-object", dtype => "string" }, # Object + { name => "FPA.OBSTYPE", type => "constant", flag => "-exp_type", dtype => "string" }, # Exposure type + { name => "FPA.FILTER", type => "constant", flag => "-filter", dtype => "string" }, # Filter used + { name => "FPA.COMMENT", type => "constant", flag => "-comment", dtype => "string" }, # Obs Comment + { name => "FPA.AIRMASS", type => "constant", flag => "-airmass", dtype => "float" }, # Airmass + { name => "FPA.RA", type => "constant", flag => "-ra", dtype => "float" }, # Right ascension + { name => "FPA.DEC", type => "constant", flag => "-decl", dtype => "float" }, # Declination + { name => "FPA.ALT", type => "constant", flag => "-alt", dtype => "float" }, # Altitude + { name => "FPA.AZ", type => "constant", flag => "-az", dtype => "float" }, # Azimuth + { name => "FPA.POSANGLE", type => "constant", flag => "-posang", dtype => "float" }, # Position angle + { name => "FPA.TIME", type => "constant", flag => "-dateobs", dtype => "string" }, # Date of observation (UTC) + { name => "FPA.TELESCOPE", type => "constant", flag => "-telescope", dtype => "string" }, # Telescope + { name => "FPA.INSTRUMENT", type => "constant", flag => "-inst", dtype => "string" }, # Instrument + { name => "FPA.M1X", type => "constant", flag => "-m1_x", dtype => "float" }, # M1X + { name => "FPA.M1Y", type => "constant", flag => "-m1_y", dtype => "float" }, # M1Y + { name => "FPA.M1Z", type => "constant", flag => "-m1_z", dtype => "float" }, # M1Z + { name => "FPA.M1TIP", type => "constant", flag => "-m1_tip", dtype => "float" }, # M1TIP + { name => "FPA.M1TILT", type => "constant", flag => "-m1_tilt", dtype => "float" }, # M1TILT + { name => "FPA.M2X", type => "constant", flag => "-m2_x", dtype => "float" }, # M2X + { name => "FPA.M2Y", type => "constant", flag => "-m2_y", dtype => "float" }, # M2Y + { name => "FPA.M2Z", type => "constant", flag => "-m2_z", dtype => "float" }, # M2Z + { name => "FPA.M2TIP", type => "constant", flag => "-m2_tip", dtype => "float" }, # M2TIP + { name => "FPA.M2TILT", type => "constant", flag => "-m2_tilt", dtype => "float" }, # M2TILT + { name => "FPA.ENV.TEMP", type => "constant", flag => "-env_temperature", dtype => "float" }, # external temp + { name => "FPA.ENV.HUMID", type => "constant", flag => "-env_humidity", dtype => "float" }, # external humidity + { name => "FPA.ENV.WIND", type => "constant", flag => "-env_wind_speed", dtype => "float" }, # external wind speed + { name => "FPA.ENV.DIR", type => "constant", flag => "-env_wind_dir", dtype => "float" }, # external wind direction + + { name => "FPA.TELTEMP.M1", type => "constant", flag => "-teltemp_m1", dtype => "float" }, # Primary mirror temps (C) + { name => "FPA.TELTEMP.M1CELL", type => "constant", flag => "-teltemp_m1cell", dtype => "float" }, # Primary mirror support temps (C) + { name => "FPA.TELTEMP.M2", type => "constant", flag => "-teltemp_m2", dtype => "float" }, # Secondary mirror temps (C + { name => "FPA.TELTEMP.SPIDER", type => "constant", flag => "-teltemp_spider", dtype => "float" }, # Spider temperatures (C) + { name => "FPA.TELTEMP.TRUSS", type => "constant", flag => "-teltemp_truss", dtype => "float" }, # Mid truss temperatures (C + { name => "FPA.TELTEMP.EXTRA", type => "constant", flag => "-teltemp_extra", dtype => "float" }, # Miscellaneous temperatures (C) + + { name => "FPA.PON.TIME", type => "constant", flag => "-pon_time", dtype => "float" }, # time since last power on + { name => "CHIP.TEMP", type => "mean", flag => "-ccd_temp", dtype => "float" }, # CCD temperature + { name => "CELL.EXPOSURE", type => "mean", flag => "-exp_time", dtype => "float" }, # Exposure time + { name => "SAT_PIXEL_FRAC", type => "mean", flag => "-sat_pixel_frac", dtype => "float" }, # fraction of saturated pixels + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('regtool') + or (warn "Can't find regtool" and $missing_tools = 1); +my $ppStats = can_run('ppStats') + or (warn "Can't find ppStats" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# setup cache interface +# XXX why is this being cached? +my $c = Cache::File->new( + cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)), + default_expires => '7200 sec', +); + +my $now_time = localtime(); +printf STDERR "\nstarting ppStats: %s\n", $now_time if $verbose; + +# Run ppStats on the input file +{ + # extract the data from the image header; -level is used to get FILE.LEVEL and CLASS.ID + my $command = "$ppStats $uri -recipe PPSTATS $RECIPE -level"; # Command to run ppStats + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform ppStats on exposure id $exp_id: $error_code", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $error_code); + } + + # Parse the output + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf); # XXX is this join necessary? + unless ($metadata) { + &my_die ("Unable to parse metadata config doc", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die ("Unable to find all values", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); + } +} + +$now_time = localtime(); +printf STDERR "\ndone with ppStats: %s\n", $now_time if $verbose; + +# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type +if (uc($stats->value_for_flag ("-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-inst")) eq "NULL") { &my_die ("inst not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-class_id")) eq "NULL") { &my_die ("class_id not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-exp_type")) eq "NULL") { &my_die ("exp_type not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } + +my $command = "$regtool -addprocessedimfile"; +$command .= " -exp_id $exp_id"; +$command .= " -exp_name $tmp_exp_name"; # keep the supplied exp_name (could be derived from the file) +$command .= " -tmp_class_id $tmp_class_id"; # the original class_id supplied by the user, replace by ppStats CLASS.ID +$command .= " -uri $uri "; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +$now_time = localtime(); +printf STDERR "\nrunning regtool update: %s\n", $now_time if $verbose; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +$now_time = localtime(); +printf STDERR "\ndone with regtool update: %s\n", $now_time if $verbose; + +sub cache_run +{ + my %p = @_; + + my $cmd_output = $c->get($p{command}) if $cache; + if (defined $cmd_output) { + return @{thaw $cmd_output}; + } else { + my @output = run(%p); + $c->set($p{command}, freeze \@output) if $cache; + return @output; + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; + my $exp_name = shift; + my $tmp_class_id = shift; + my $uri = shift; + my $exit_code = shift; + + # for failed imfiles, we insert UNKNOWN for inst, telescope, class_id + + carp($msg); + if (defined $exp_id && defined $tmp_class_id and not $no_update) { + my $command = "$regtool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -exp_name $exp_name"; + $command .= " -tmp_class_id $tmp_class_id"; + $command .= " -uri $uri "; + $command .= " -telescope UNKNOWN"; + $command .= " -inst UNKNOWN"; + $command .= " -class_id $tmp_class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/stack_skycell.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/stack_skycell.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/stack_skycell.pl (revision 22293) @@ -0,0 +1,261 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use File::Temp qw( tempfile ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($stack_id, $dbname, $outroot, $verbose, $no_update, $no_op, $save_temps); +GetOptions( + 'stack_id|d=s' => \$stack_id, # Stack identifier + 'dbname|d=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --stack_id", + -exitval => 3, + ) unless defined $stack_id + and defined $outroot; + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC STACKTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, +# { name => "DT_STACK", type => "sum", flag => "-dtime_stack", dtype => "float" }, + { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $ppStack = can_run('ppStack') or (warn "Can't find ppStack" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of components for stacking +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$stacktool -inputskyfile -stack_id $stack_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -inputskyfile: $error_code", $stack_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $stack_id, $PS_EXIT_PROG_ERROR); +} + +&my_die("Stack list contains less than two elements", $stack_id, $PS_EXIT_SYS_ERROR) unless + scalar @$files >= 2; + +# Parse the list of input files to get the tesselation, skycell identifiers and camera +my $skycell_id; # Skycell identifier +my $tess_id; # Tesselation identifier +my $camera; # Camera +foreach my $file (@$files) { + # skip warps which are speicified as 'ignored' + if ($file->{ignored}) { next; } + if (defined $tess_id) { + &my_die("Tesselation identifiers don't match", $stack_id, $PS_EXIT_SYS_ERROR) unless + $file->{tess_id} eq $tess_id; + } else { + $tess_id = $file->{tess_id}; + } + if (defined $skycell_id) { + &my_die("Skycell identifiers don't match", $stack_id, $PS_EXIT_SYS_ERROR) unless + $file->{skycell_id} eq $skycell_id; + } else { + $skycell_id = $file->{skycell_id}; + } + if (defined $camera) { + &my_die("Cameras don't match", $stack_id, $PS_EXIT_SYS_ERROR) unless $file->{camera} eq $camera; + } else { + $camera = $file->{camera}; + } +} + +&my_die("Can't find camera", $stack_id, $PS_EXIT_SYS_ERROR) unless defined $camera; +$ipprc->define_camera($camera); + +# Generate MDC file with the inputs +my ($listFile, $listName) = tempfile( "$tess_id.$skycell_id.stk$stack_id.list.XXXX", + UNLINK => !$save_temps ); +my $num = 0; +my $inputSources; # Sources to use as stamps +foreach my $file (@$files) { + if ($file->{ignored}) { next; } + + print $listFile "INPUT$num\tMETADATA\n"; + $num++; + + my $image = $file->{uri}; # Image name + my $mask = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name + my $weight = $ipprc->filename( "PSWARP.OUTPUT.WEIGHT", $file->{path_base} ); # Weight name + my $psf = $ipprc->filename( "PSPHOT.PSF.SAVE", $file->{path_base} ); # Weight name + + &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image ); + &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask ); + &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $weight ); + &my_die("PSF $psf does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $psf ); + + print $listFile "\tIMAGE\tSTR\t" . $image . "\n"; + print $listFile "\tMASK\tSTR\t" . $mask . "\n"; + print $listFile "\tWEIGHT\tSTR\t" . $weight . "\n"; + print $listFile "\tPSF\tSTR\t" . $psf . "\n"; + + ### XXX NEED TO UPDATE THESE appropriately + print $listFile "\tWEIGHTING\tF32\t" . 1.0 . "\n"; + + print $listFile "END\n\n"; + + # XXX details about supplying sources, etc, need to be specified in the libraries + unless (defined $inputSources) { + $inputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); + &my_die("Source file $inputSources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $inputSources ); + } + +} + +# Get the output filenames +my $outputName = $ipprc->filename("PPSTACK.OUTPUT", $outroot); +my $outputMask = $ipprc->filename("PPSTACK.OUTPUT.MASK", $outroot); +my $outputWeight = $ipprc->filename("PPSTACK.OUTPUT.WEIGHT", $outroot); +my $outputSources = $ipprc->filename("PSPHOT.OUTPUT", $outroot, "none"); +#my $bin1Name = $ipprc->filename("PPSTACK.BIN1", $outroot); +#my $bin2Name = $ipprc->filename("PPSTACK.BIN2", $outroot); +my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); +my $logDest = $ipprc->filename("LOG.EXP", $outroot); + +# Perform stacking +unless ($no_op) { + my $command = "$ppStack $listName $outroot"; + $command .= " -stats $outputStats"; + $command .= " -recipe PPSUB STACK"; + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -sources $inputSources"; + $command .= " -photometry"; + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStack: $error_code", $stack_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + &my_die("Couldn't find expected output file: $outputMask", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + &my_die("Couldn't find expected output file: $outputSources", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); +# &my_die("Couldn't find expected output file: $bin1Name", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name); +# &my_die("Couldn't find expected output file: $bin2Name", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name); + &my_die("Couldn't find expected output file: $outputStats", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + + # Get the statistics on the stacked image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $stack_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $metadata = $mdcParser->parse(join "", @contents) or + &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $stack_id, $PS_EXIT_PROG_ERROR); +} + +unless ($no_update) { + + # Add the stack result + { + my $command = "$stacktool -addsumskyfile -stack_id $stack_id -uri $outputName -path_base $outroot"; + $command .= $stats->cmdflags(); + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -addsumskyfile: $error_code", $stack_id, $error_code); + } + } + + # Register the run as completed + { + my $command = "$stacktool -updaterun -stack_id $stack_id -state stop"; # Command to run stacktool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform stacktool -updaterun: $error_code\n"); + exit($error_code); + } + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $stack_id = shift; # Stack identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if (defined $stack_id and not $no_update) { + my $command = "$stacktool -addsumskyfile -stack_id $stack_id -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/summit_copy.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/summit_copy.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/summit_copy.pl (revision 22293) @@ -0,0 +1,159 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ( $uri, $filename, $compress, $bytes, $md5, $nebulous, $exp_name, $inst, $telescope, $class, $class_id, $end_stage, $workdir, + $dbname, $verbose, $no_update, $no_op ); +GetOptions( + 'uri=s' => \$uri, # source location of file on data store + 'filename=s' => \$filename, # target location of file on local system + 'compress' => \$compress, # request file in compressed format + 'bytes=s' => \$bytes, # reported file size in bytes + 'md5=s' => \$md5, # reported md5 checksum + 'nebulous' => \$nebulous, # use nebulous for the target file + 'exp_name=s' => \$exp_name, # Exposure name + 'inst=s' => \$inst, # Instrument + 'telescope=s' => \$telescope, # Telescope + 'class=s' => \$class, # Class level + 'class_id=s' => \$class_id, # Class identifier + 'end_stage=s' => \$end_stage, # end of processing + 'workdir=s' => \$workdir, # workdir + 'dbname=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --uri --filename --exp_name --inst --telescope --class --class_id --workdir", + -exitval => 3) unless + defined $uri and + defined $filename and + defined $exp_name and + defined $inst and + defined $telescope and + defined $class and + defined $class_id and + defined $workdir; + +# Look for programs we need +my $missing_tools; +my $dsget = can_run('dsget') or (warn "Can't find dsget" and $missing_tools = 1); +my $pztool = can_run('pztool') or (warn "Can't find pztool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $command; + +# dsget command +$command = "$dsget --uri $uri --filename $filename"; +$command .= " --compress" if defined $compress; +$command .= " --bytes $bytes" if defined $bytes; +$command .= " --nebulous" if defined $nebulous; +$command .= " --md5 $md5" if defined $md5; + +# run command +unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dsget: $error_code", $exp_name, $inst, $telescope, $class, $class_id, $uri, $error_code); + } +} else { + print "skipping command: $command\n"; +} + +# command to update database +$command = "$pztool -copydone"; +$command .= " -exp_name $exp_name"; +$command .= " -inst $inst"; +$command .= " -telescope $telescope"; +$command .= " -class $class"; +$command .= " -class_id $class_id"; +$command .= " -uri $filename"; +$command .= " -workdir $workdir"; +$command .= " -end_stage $end_stage" if defined $end_stage; +$command .= " -dbname $dbname" if defined $dbname; + +# update the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform $command: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_name = shift; # Chiptool identifier + my $inst = shift; # Chiptool identifier + my $telescope = shift; # Class identifier + my $class = shift; # Class identifier + my $class_id = shift; # Class identifier + my $uri = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + unless ($no_update) { + # command to update database + my $command; + $command = "$pztool -copydone"; + $command .= " -exp_name $exp_name"; + $command .= " -inst $inst"; + $command .= " -telescope $telescope"; + $command .= " -class $class"; + $command .= " -class_id $class_id"; + $command .= " -uri $uri"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/warp_overlap.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/warp_overlap.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/warp_overlap.pl (revision 22293) @@ -0,0 +1,316 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); +use File::Spec; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + metadataLookupStr + metadataLookupBool + caturi + file_scheme + ); +use PS::IPP::Operations qw( generate_skycells ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($warp_id, $camera, $tess_id, $dbname, $verbose, $no_update, $no_op); +GetOptions( + 'warp_id|i=s' => \$warp_id, # Warp identifier + 'camera|c=s' => \$camera, # Camera name + 'tess_id=s' => \$tess_id, # Tessellation identifier + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --warp_id --camera --tess_id", + -exitval => 3, +) unless defined $warp_id + and defined $camera + and defined $tess_id; + +$ipprc->define_camera($camera); + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $dvoImageOverlaps = can_run('dvoImageOverlaps') or (warn "Can't find dvoImageOverlaps" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get list of component imfiles for exposure +my $imfiles; +{ + my $command = "$warptool -imfile -warp_id $warp_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -imfile: $error_code", $warp_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $warp_id, $PS_EXIT_PROG_ERROR); +} + +# Where do we get the astrometry source from? +my $astromSource; # The astrometry source +my $astromAccept; # Accept the astrometry unconditionally? +my $astromDepth; # File level of the astrometry source (SPLIT or MEF)? +{ + my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); + $astromAccept = metadataLookupBool($metadata, 'ASTROM.ACCEPT'); + $astromDepth = metadataLookupStr($metadata, 'ASTROM.DEPTH'); +} + +# Determine the imfile/skycell overlaps +my @overlaps = (); + +unless ($no_op) { + # Calculate the overlaps between imfiles and skycells + + my $tess_dir = $ipprc->tessellation_catdir( $tess_id ); # Tessellation catdir for DVO + $tess_dir = $ipprc->convert_filename_absolute( $tess_dir ); + + my %unique_skycells = (); # Identified skycells (all unique by virtue of hash property) + + # XXX this is a bit too hard wired: the concept is that astrometry comes from the MOSAIC vs CHIP output + if ($astromSource eq 'PSASTRO.OUTPUT.MEF') { + # We have a MEF astrometry file from psastro + my $imfile = $imfiles->[0]; + my $camRoot = $imfile->{cam_path_base}; + my $astromFile = $ipprc->filename($astromSource, $camRoot); # Astrometry file + + my @matchlist = get_overlaps($astromFile, $tess_dir, $astromAccept); # List of overlaps + if (! @matchlist) { + &my_die("Unable to perform dvoImageOverlaps: missing astrometry", $warp_id, $PS_EXIT_DATA_ERROR); + } + # Match each of the imfiles to this list + foreach my $imfile (@$imfiles) { + extract_overlaps(\@matchlist, $imfile, $astromFile, $tess_id, \@overlaps, \%unique_skycells); + } + } else { + # We have per-imfile astrometry + foreach my $imfile (@$imfiles) { + my $astromFile; + if ($astromSource eq 'PSASTRO.OUTPUT') { + my $chipRoot = $imfile->{chip_path_base}; + my $classID = $imfile->{class_id}; + $astromFile = $ipprc->filename($astromSource, $chipRoot, $classID); # Astrometry file + } else { + $astromFile = $imfile->{chip_uri}; # Astrometry file + } + my @matchlist = get_overlaps($astromFile, $tess_dir, $astromAccept); # List of overlaps + if (! @matchlist and $verbose) { + print "skipping $astromFile\n"; + } + + extract_overlaps(\@matchlist, $imfile, $astromFile, $tess_id, \@overlaps, \%unique_skycells); + } + } +} else { + # create an overlap with an entry for each skycell:imfile match + foreach my $imfile (@$imfiles) { + my %overlap = (); + $overlap{skycell_id} = 'default'; + $overlap{tess_id} = 'default'; + $overlap{cam_id} = $imfile->{cam_id}; + $overlap{class_id} = $imfile->{class_id}; + $overlap{fault} = $imfile->{fault}; + push @overlaps, \%overlap; + } +} + +# XXX this file needs some additional error checking: if no overlaps are found, we +# keep running this step over and over (a successful warptool -addoverlap prevents +# successive warptime -imfile from running. + +# Generate a MDC file with the overlaps +my ($overlapFile, $overlapName) = tempfile( 'overlaps.wrp' . $warp_id . '.mdc.XXXX', UNLINK => 1 ); +print $overlapFile "warpSkyCellMap MULTI\n\n"; +foreach my $overlap (@overlaps) { + print $overlapFile "warpSkyCellMap METADATA\n"; + print $overlapFile " warp_id S32 $warp_id\n"; + print $overlapFile " skycell_id STR $overlap->{skycell_id}\n"; + print $overlapFile " tess_id STR $overlap->{tess_id}\n"; + print $overlapFile " cam_id S32 $overlap->{cam_id}\n"; + print $overlapFile " class_id STR $overlap->{class_id}\n"; + print $overlapFile " fault S16 $overlap->{fault}\n"; + print $overlapFile "END\n\n"; +} +close $overlapFile; + +system "cat $overlapName" if $verbose; + +# Add the processed file to the database +unless ($no_update) { + my $command = "$warptool -addoverlap -mapfile $overlapName"; # Command to run warptool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addoverlap: $error_code\n"); + exit($error_code); + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $warp_id = shift; # Warp identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + + # No failure option on warptool for this mode, so just die. + exit $exit_code; +} + +# Run dvoImageOverlaps to get the overlaps; return the output +sub get_overlaps +{ + my $filename = shift; # Filename on which to run dvoImageOverlaps + my $tess_dir = shift; # Tessellation directory + my $accept = shift; # Do we use the -accept-astrom flag? + + my $command = "$dvoImageOverlaps -D CATDIR $tess_dir " . $ipprc->file_resolve($filename); + $command .= ' -accept-astrom' if $accept; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + if (!$success) { + print "Missing astrometry for $filename --- check ASTROM.SOURCE in PSWARP recipe\n"; + return 0; + } + return split ('\n', (join "", @$stdout_buf)); +} + +# Extract a list of overlaps for an imfile +# +# The command "dvoImageOverlaps -accept-astrom -D CATDIR tessellation (megacam)" returns: +# 729534pa.cmf[ccd00.hdr] : skycell.051.fits +# PSASTRO.OUTPUT[CMF.HEAD] : SKYCELL +# [CMF.HEAD] is optionally used for MEF files (SIMPLE or MOSAIC) +sub extract_overlaps +{ + my $matches = shift; # Reference to list of skycells from dvoImageOverlaps + my $imfile = shift; # Imfile information + my $filename = shift; # Filename used with dvoImageOverlaps + my $tess_id = shift; # Tessellation identifier + my $overlaps = shift; # Reference to list of overlaps + my $unique_skycells = shift; # Reference to hash of found skycells + + # Get rid of the path + my @dirlist = File::Spec->splitdir( $filename ); # The elements of the full path + $filename = pop @dirlist; + + # Work out how to identify this imfile in the output + my $fileLevel = $imfile->{filelevel}; + my $entry; # How to identify this imfile in the dvoImageOverlaps output + if ((lc($fileLevel) eq "chip") && (lc($astromDepth) eq "MEF")) { + my $class_id = $imfile->{class_id}; + my $chipRoot = $ipprc->file_resolve( $imfile->{chip_path_base} ); + my $extname = $ipprc->extname_rule("CMF.HEAD", $class_id); # MEF psastro output + + + $entry = $filename . '\[' . $extname . '\]'; + print STDERR "entry: $entry, class: $class_id, extname: $extname, chiproot: $chipRoot\n" if $verbose; + } else { + $entry = $filename; + print STDERR "entry: $entry\n" if $verbose; + } + + my @skycells = &select_skycells($entry, @$matches); # Matching skycells + my $Nskycells = @skycells; + printf STDERR "Nskycells: $Nskycells\n" if $verbose; + foreach my $skycell (@skycells) { + my %overlap = (); # Overlap information for warptool + $overlap{skycell_id} = $skycell; + $overlap{tess_id} = $tess_id; + $overlap{cam_id} = $imfile->{cam_id}; + $overlap{class_id} = $imfile->{class_id}; + $overlap{fault} = $imfile->{fault}; + push @$overlaps, \%overlap; + + printf STDERR "overlap: %s : %s , %s\n", $skycell, $imfile->{cam_id}, $imfile->{class_id} if $verbose; + + $unique_skycells->{$skycell} = 1; + } + + return; +} + +# Find skycells in the list that come from a particular entry +sub select_skycells +{ + my $entry = shift; # File+Ext to search for + my @list = @_; # List of "File+Ext : skycell" + + my @skycells = (); + my %unique = (); # Ensure we only return unique skycells for this entry + + foreach my $line (@list) { + if ($line =~ m|$entry|) { + my ($skycell) = $line =~ m|$entry\S*\s+:\s+(\S+)|; + if (not defined $unique{$skycell}) { + push @skycells, $skycell; + $unique{$skycell} = 1; + } + } + } + return @skycells; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippScripts/scripts/warp_skycell.pl =================================================================== --- /tags/pap_tags/pap_root_080320/ippScripts/scripts/warp_skycell.pl (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippScripts/scripts/warp_skycell.pl (revision 22293) @@ -0,0 +1,255 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + metadataLookupStr + metadataLookupBool + caturi + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($warp_id, $skycell_id, $tess_id, $camera, $dbname, $outroot, $verbose, $no_update, $no_op, $save_temps); +GetOptions( + 'warp_id|i=s' => \$warp_id, # Warp identifier + 'skycell_id|s=s' => \$skycell_id, # Skycell identifier + 'tess_id|s=s' => \$tess_id, # Tesselation identifier + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --warp_id --skycell_id --tess_id --camera --outroot", + -exitval => 3, +) unless defined $warp_id + and defined $skycell_id + and defined $tess_id + and defined $camera + and defined $outroot; + +$ipprc->define_camera($camera); + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC WARPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, +# { name => "DT_WARP", type => "sum", flag => "-dtime_warp", dtype => "float" }, + { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $pswarp = can_run('pswarp') or (warn "Can't find pswarp" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component imfiles for exposure +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $imfiles; +{ + my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -scmap: $error_code", $warp_id, $skycell_id, $tess_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_id, $PS_EXIT_PROG_ERROR); + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $warp_id, $skycell_id, $tess_id, $PS_EXIT_PROG_ERROR); +} + +# Where do we get the astrometry source from? +my $astromSource; # The astrometry source +{ + my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); +} + +my $outputImage = $ipprc->filename("PSWARP.OUTPUT", $outroot, $skycell_id ); +my $outputMask = $ipprc->filename("PSWARP.OUTPUT.MASK", $outroot, $skycell_id); +my $outputWeight = $ipprc->filename("PSWARP.OUTPUT.WEIGHT", $outroot, $skycell_id); +my $outputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id); +my $outputPSF = $ipprc->filename("PSPHOT.PSF.SAVE", $outroot, $skycell_id); +my $outputBin1 = $ipprc->filename("PSWARP.BIN1", $outroot, $skycell_id ); +my $outputBin2 = $ipprc->filename("PSWARP.BIN2", $outroot, $skycell_id ); +my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot, $skycell_id ); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id); +my $logDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id); + +my $skyFile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id ); +$ipprc->skycell_file( $tess_id, $skycell_id, $skyFile, $verbose ) or &my_die("Unable to generate template skycell", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR); + +# Get list of filenames +my $outrootResolved = $ipprc->file_resolve( $outroot ); +my ($imageFile, $imageName) = tempfile( "$outrootResolved.image.list.XXXX", UNLINK => !$save_temps); +my ($maskFile, $maskName) = tempfile( "$outrootResolved.mask.list.XXXX", UNLINK => !$save_temps); +my ($weightFile, $weightName) = tempfile( "$outrootResolved.weight.list.XXXX", UNLINK => !$save_temps); +my ($astromFile, $astromName) = tempfile( "$outrootResolved.astrom.list.XXXX", UNLINK => !$save_temps); + +foreach my $imfile (@$imfiles) { + my $image = $imfile->{uri}; # Image name + my $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name + my $weight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name + + &my_die("Couldn't find input file: $image", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image); + &my_die("Couldn't find input file: $mask", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($mask); + + my $astrom; # Astrometry file + if ($astromSource eq 'PSASTRO.OUTPUT.MEF') { + $astrom = $ipprc->filename($astromSource, $imfile->{cam_path_base}); + } elsif ($astromSource eq 'PSASTRO.OUTPUT') { + my $chipRoot = $imfile->{chip_path_base}; + my $classID = $imfile->{class_id}; + $astrom = $ipprc->filename($astromSource, $chipRoot, $classID); # Astrometry file + } + + &my_die("Couldn't find input file: $astrom", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless defined $astrom and $ipprc->file_exists($astrom); + + print $imageFile "$image\n"; + print $maskFile "$mask\n"; + print $weightFile "$weight\n"; + print $astromFile "$astrom\n"; +} +close $imageFile; +close $maskFile; +close $weightFile; +close $astromFile; + + +# Run pswarp +my $accept = 1; # Accept the skycell? +unless ($no_op) { + my $command = "$pswarp"; + $command .= " -list $imageName"; + $command .= " -masklist $maskName"; + $command .= " -weightlist $weightName"; + $command .= " -astromlist $astromName"; + $command .= " $outroot $skyFile"; + $command .= " -stats $outputStats"; + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -psf"; # Turn on PSF determination + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform pswarp: $error_code", $warp_id, $skycell_id, $tess_id, $error_code); + } + + # Check first for the stats file, and if the ACCEPT flag is set. + &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + # Get the statistics on the warped image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or die "Can't open statistics file $outputStats: $!\n"; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $contents = join "", @contents; + + my $metadata = $mdcParser->parse($contents) + or &my_die("Unable to parse metadata config", $warp_id, $skycell_id, $tess_id, $PS_EXIT_PROG_ERROR); + $accept = metadataLookupBool($metadata, "ACCEPT"); + if ($accept) { + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $warp_id, $skycell_id, $tess_id, $PS_EXIT_PROG_ERROR); + + &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); + &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputPSF); +# &my_die("Couldn't find expected output file: $outputBin1", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1); +# &my_die("Couldn't find expected output file: $outputBin2", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2); + } +} + +unless ($no_update) { + my $command = "$warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id"; + $command .= " -ignore" if not $accept; # Completed succesfully, but can't produce product + $command .= " -uri $outputImage -path_base $outroot" if $accept; + $command .= $stats->cmdflags() if $accept; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addwarped: $error_code\n"); + exit($error_code); + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $warp_id = shift; # Warp identifier + my $skycell_id = shift; # Skycell identifier + my $tess_id = shift; # Tesselation identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if (defined $warp_id and defined $skycell_id and defined $tess_id and not $no_update) { + my $command = "$warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_tags/pap_root_080320/ippconfig/cfh12k/filerules-mef.mdc =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/cfh12k/filerules-mef.mdc (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/cfh12k/filerules-mef.mdc (revision 22293) @@ -0,0 +1,179 @@ +### File rules for PHU=FPA, EXT=CHIP + +### Redirections +PPIMAGE.OUTPUT STR PPIMAGE.OUTPUT.MEF +PPIMAGE.OUTPUT.MASK STR PPIMAGE.OUT.MK.MEF +PPIMAGE.OUTPUT.WEIGHT STR PPIMAGE.OUT.WT.MEF + +PPIMAGE.CHIP STR PPIMAGE.CHIP.MEF +PPIMAGE.CHIP.MASK STR PPIMAGE.CHIP.MK.MEF +PPIMAGE.CHIP.WEIGHT STR PPIMAGE.CHIP.WT.MEF + +PPIMAGE.OUTPUT.FPA1 STR PPIMAGE.OUTPUT.FPA1.MEF +PPIMAGE.OUTPUT.FPA2 STR PPIMAGE.OUTPUT.FPA2.MEF +PPIMAGE.BIN1 STR PPIMAGE.BIN1.MEF +PPIMAGE.BIN2 STR PPIMAGE.BIN2.MEF + +PPSTAMP.CHIP STR PPSTAMP.CHIP.MEF + +PSASTRO.INPUT STR PSASTRO.INPUT.CMF +PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.MEF +PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF +PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.MEF + +DVOCORR.OUTPUT STR DVOCORR.MEF.OUTPUT +DVOFLAT.OUTPUT STR DVOFLAT.MEF.OUTPUT + +### input file definitions +### use @DETDB entries to get the detrend images from the database +### replace @DETDB with @FILES if you want to require it from the +### command line, or with an explicit name to require a specific file +TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + +## files used by ppImage +PPIMAGE.INPUT INPUT @FILES CHIP IMAGE +PPIMAGE.MASK INPUT @DETDB CHIP MASK +PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE +PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE +PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE +PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE + +## files used to build and apply the flat-field correction images +DVOCORR.INPUT INPUT @FILES CHIP IMAGE +DVOCORR.REFHEAD INPUT @FILES CHIP HEADER +DVOFLAT.INPUT INPUT @FILES CHIP IMAGE +DVOFLAT.CORR INPUT @DETDB CHIP IMAGE + +## files used by psphot +PSPHOT.LOAD INPUT @FILES CHIP IMAGE +PSPHOT.INPUT INPUT @FILES CHIP IMAGE +PSPHOT.MASK INPUT @FILES CHIP MASK +PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT +PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF +PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by psastro +PSASTRO.INPUT.CMP INPUT @FILES CHIP CMP +PSASTRO.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by pswarp +PSWARP.INPUT INPUT @FILES CHIP IMAGE +PSWARP.WEIGHT INPUT @FILES CHIP WEIGHT +PSWARP.MASK INPUT @FILES CHIP MASK +PSWARP.SKYCELL INPUT @FILES CHIP IMAGE +PSWARP.ASTROM INPUT @FILES CHIP CMF + +PPSUB.INPUT INPUT @FILES FPA IMAGE +PPSUB.INPUT.MASK INPUT @FILES FPA MASK +PPSUB.INPUT.WEIGHT INPUT @FILES FPA WEIGHT +PPSUB.REF INPUT @FILES FPA IMAGE +PPSUB.REF.MASK INPUT @FILES FPA MASK +PPSUB.REF.WEIGHT INPUT @FILES FPA WEIGHT +PPSUB.SOURCES INPUT @FILES FPA CMF + +PPSTACK.INPUT INPUT @FILES FPA IMAGE +PPSTACK.INPUT.MASK INPUT @FILES FPA MASK +PPSTACK.INPUT.WEIGHT INPUT @FILES FPA WEIGHT +PPSTACK.SOURCES INPUT @FILES FPA CMF + +PPSTAMP.INPUT INPUT @FILES CHIP IMAGE + +PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE +PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + +### output file definitions +TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT +PPIMAGE.OUTPUT.MEF OUTPUT {OUTPUT}.b0.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.OUT.MK.MEF OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK CHIP TRUE MEF +PPIMAGE.OUT.WT.MEF OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT CHIP TRUE MEF +PPIMAGE.OUTPUT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b0.fits IMAGE NONE CHIP TRUE SPLIT +PPIMAGE.OUT.MK.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits MASK NONE CHIP TRUE SPLIT +PPIMAGE.OUT.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT NONE CHIP TRUE SPLIT + +PPIMAGE.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.CHIP.MK.MEF OUTPUT {OUTPUT}.ch.mk.fits MASK COMP_MASK CHIP TRUE MEF +PPIMAGE.CHIP.WT.MEF OUTPUT {OUTPUT}.ch.wt.fits WEIGHT COMP_WT CHIP TRUE MEF +PPIMAGE.CHIP.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE NONE CHIP TRUE SPLIT +PPIMAGE.CHIP.MK.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.mk.fits MASK NONE CHIP TRUE SPLIT +PPIMAGE.CHIP.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.wt.fits WEIGHT NONE CHIP TRUE SPLIT + +PPIMAGE.OUTPUT.FPA1.MEF OUTPUT {OUTPUT}.b1.fits IMAGE COMP_IMG FPA TRUE MEF +PPIMAGE.OUTPUT.FPA2.MEF OUTPUT {OUTPUT}.b2.fits IMAGE COMP_IMG FPA TRUE MEF +PPIMAGE.OUTPUT.FPA1.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE NONE FPA TRUE SPLIT +PPIMAGE.OUTPUT.FPA2.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE NONE FPA TRUE SPLIT + +PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE MEF + +PPIMAGE.BIN1.MEF OUTPUT {OUTPUT}.b1c.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.BIN2.MEF OUTPUT {OUTPUT}.b2c.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.BIN1.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE COMP_IMG CHIP TRUE SPLIT +PPIMAGE.BIN2.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE COMP_IMG CHIP TRUE SPLIT + +PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE + +PPMERGE.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE MEF + +DVOCORR.MEF.OUTPUT OUTPUT {OUTPUT}.fc.fits IMAGE NONE CHIP TRUE MEF +DVOCORR.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE NONE CHIP TRUE SPLIT +DVOFLAT.MEF.OUTPUT OUTPUT {OUTPUT}.co.fits IMAGE NONE CHIP TRUE MEF +DVOFLAT.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE NONE CHIP TRUE SPLIT + +PSPHOT.RESID OUTPUT {OUTPUT}.res.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKGND OUTPUT {OUTPUT}.bck.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKSUB OUTPUT {OUTPUT}.sub.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKMDL OUTPUT {OUTPUT}.mdl.fits IMAGE NONE CHIP FALSE MEF + +PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW NONE CHIP TRUE NONE +PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.{CHIP.NAME}.sx SX NONE CHIP TRUE NONE +PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.{CHIP.NAME}.obj OBJ NONE CHIP TRUE NONE +PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.cmp CMP NONE CHIP TRUE NONE +PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF NONE CHIP TRUE SPLIT +PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF NONE CHIP TRUE MEF + +PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE CHIP TRUE MEF + +SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.mnt.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.psf.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.dap.png KAPA NONE FPA TRUE NONE + +PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE +PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE SPLIT +PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUT.MODEL OUTPUT {OUTPUT}.asm ASTROM.MODEL NONE FPA TRUE NONE +PSASTRO.OUT.REFSTARS OUTPUT {OUTPUT}.aref.fits ASTROM.REFSTARS NONE FPA TRUE NONE + +PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT FPA TRUE NONE +PSWARP.OUTPUT.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE +PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE COMP_IMG FPA TRUE NONE +PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE COMP_IMG FPA TRUE NONE + +SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE +SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + +PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT FPA TRUE NONE + +PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT FPA TRUE NONE + +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + +PPSIM.OUTPUT.MEF OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE MEF +PPSIM.OUTPUT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT +PPSIM.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE MEF + +PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE COMP_IMG CHIP TRUE NONE +PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK COMP_MASK CHIP TRUE NONE + +LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE +LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE + +TRACE.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.trace TEXT NONE CHIP TRUE NONE +TRACE.EXP OUTPUT {OUTPUT}.trace TEXT NONE FPA TRUE NONE Index: /tags/pap_tags/pap_root_080320/ippconfig/ctio_mosaic2/camera.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/ctio_mosaic2/camera.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/ctio_mosaic2/camera.config (revision 22293) @@ -0,0 +1,403 @@ +# Camera configuration file for the CTIO MOSAIC2 : describes the camera + +# File formats that we know about +FORMATS METADATA + MEF STR ctio_mosaic2/format.config +END + +# Description of camera --- all the chips and the cells that comprise them +FPA METADATA + ccd00 STR LeftAmp RightAmp + ccd01 STR LeftAmp RightAmp + ccd02 STR LeftAmp RightAmp + ccd03 STR LeftAmp RightAmp + ccd04 STR LeftAmp RightAmp + ccd05 STR LeftAmp RightAmp + ccd06 STR LeftAmp RightAmp + ccd07 STR LeftAmp RightAmp +END + +# menu of possible filter names: internal STR external name +FILTER.ID METADATA + VR MULTI + VR STR VR.Supermacho.c6027 + VR STR VR Supermacho c6027 + VR STR VR SuperMacho c6027 + R STR R Harris c6004 + I STR I c6028 + U STR U c6001 + B STR B Harris c6002 + V STR V Harris c6026 + # Minor filters + g MULTI + g STR g SDSS c6017 + g STR SDSS g c6017 + r MULTI + r STR SDSS r c6018 + r STR r SDSS c6018 + i MULTI + i STR i SDSS c6019 + i STR SDSS i c6019 + # Unused filters (only used in zeros, and therefore irrelevent) + NULL MULTI + NULL STR z SDSS c6020 + NULL STR SDSS z c6020 + NULL STR ha8 H-alpha+8nm c6011 + NULL STR M Washington c6007 + NULL STR u SDSS c6021 + NULL STR VRr Supermacho c6027 +END + +# Table of strings to use for the class identifier (see ippdb) according to the file level. +CLASSID METADATA + FPA STR fpa +END + +DVO.CAMERADIR STR ctio_mosaic2 # Camera directory for DVO + +# convert supplied FPA.OBSTYPE values to abstract exptype names +OBSTYPE.TABLE METADATA + bias STR BIAS + zero STR BIAS + dark STR DARK + flat STR SKYFLAT + skyflat STR SKYFLAT + domeflat STR DOMEFLAT + object STR OBJECT + science STR OBJECT +END + +# Recipe options +RECIPES METADATA + # Recipes for ppImage + PPIMAGE STR ctio_mosaic2/ppImage.config # Default: all (normal) options on + PSPHOT STR ctio_mosaic2/psphot.config # Default: all (normal) options on + PPMERGE STR ctio_mosaic2/ppMerge.config +END + +# Reduction classes +REDUCTION METADATA + # Detrend processing + DETREND METADATA + BIAS_PROCESS STR PPIMAGE_O + BIAS_RESID STR PPIMAGE_B + BIAS_VERIFY STR PPIMAGE_OB + BIAS_STACK STR PPMERGE_BIAS + DARK_PROCESS STR PPIMAGE_OB + DARK_RESID STR PPIMAGE_D + DARK_VERIFY STR PPIMAGE_OBD + DARK_STACK STR PPMERGE_DARK + SHUTTER_PROCESS STR PPIMAGE_OBD + SHUTTER_RESID STR PPIMAGE_S + SHUTTER_VERIFY STR PPIMAGE_OBDS + SHUTTER_STACK STR PPMERGE_SHUTTER + FLAT_PROCESS STR PPIMAGE_OBDS + FLAT_RESID STR PPIMAGE_F + FLAT_VERIFY STR PPIMAGE_OBDSF + FLAT_STACK STR PPMERGE_FLAT + FRINGE_PROCESS STR PPIMAGE_OBDSF + FRINGE_RESID STR PPIMAGE_R + FRINGE_VERIFY STR PPIMAGE_OBDSFR + FRINGE_STACK STR PPMERGE_FRINGE + + # Generation of pixel masks from darks and flats + DARKMASK_PROCESS STR PPIMAGE_OBD + DARKMASK_RESID STR PPIMAGE_N + DARKMASK_VERIFY STR PPIMAGE_OBD + DARKMASK_STACK STR PPMERGE_DARKMASK + FLATMASK_PROCESS STR PPIMAGE_OBDSF + FLATMASK_RESID STR PPIMAGE_N + FLATMASK_VERIFY STR PPIMAGE_OBDSF + FLATMASK_STACK STR PPMERGE_FLATMASK + JPEG_BIN1_IMAGE_DARKMASK STR PPIMAGE_J1_IMAGE_M + JPEG_BIN2_IMAGE_DARKMASK STR PPIMAGE_J2_IMAGE_M + JPEG_BIN1_IMAGE_FLATMASK STR PPIMAGE_J1_IMAGE_M + JPEG_BIN2_IMAGE_FLATMASK STR PPIMAGE_J2_IMAGE_M + JPEG_BIN1_RESID_DARKMASK STR PPIMAGE_J1_RESID_M + JPEG_BIN2_RESID_DARKMASK STR PPIMAGE_J2_RESID_M + JPEG_BIN1_RESID_FLATMASK STR PPIMAGE_J1_RESID_M + JPEG_BIN2_RESID_FLATMASK STR PPIMAGE_J2_RESID_M + + JPEG_BIN1_IMAGE_BIAS STR PPIMAGE_J1_IMAGE_B + JPEG_BIN1_IMAGE_DARK STR PPIMAGE_J1_IMAGE_B + JPEG_BIN1_IMAGE_SHUTTER STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_FLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_DOMEFLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_SKYFLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_FRINGE STR PPIMAGE_J1_IMAGE_R + JPEG_BIN2_IMAGE_BIAS STR PPIMAGE_J2_IMAGE_B + JPEG_BIN2_IMAGE_DARK STR PPIMAGE_J2_IMAGE_B + JPEG_BIN2_IMAGE_SHUTTER STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_FLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_DOMEFLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_SKYFLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_FRINGE STR PPIMAGE_J2_IMAGE_R + + JPEG_BIN1_RESID_BIAS STR PPIMAGE_J1_RESID_B + JPEG_BIN1_RESID_DARK STR PPIMAGE_J1_RESID_B + JPEG_BIN1_RESID_SHUTTER STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_FLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_DOMEFLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_SKYFLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_FRINGE STR PPIMAGE_J1_RESID_R + JPEG_BIN2_RESID_BIAS STR PPIMAGE_J2_RESID_B + JPEG_BIN2_RESID_DARK STR PPIMAGE_J2_RESID_B + JPEG_BIN2_RESID_SHUTTER STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_FLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_DOMEFLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_SKYFLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_FRINGE STR PPIMAGE_J2_RESID_R + END + # Processing raw data + DEFAULT METADATA + CHIP STR PPIMAGE_OBDSFRA + END +END + +FITS METADATA +# BITPIX is the bits per pixel for writing the output data +# COMP = NONE|RICE|GZIP|HCOMPRESS|PLIO is the compression algorithm +# TILE.[XYZ] are the tile sizes. 0 means entire the dimension, so (0,1,1) forms tiles from rows +# NOISE [0..16] is the number of "noise bits" to preserve when quantising floating point data; 16 for no loss +# HSCALE is the scale factor for lossy compression with HCOMPRESS; 0 or 1 for none; 2*RMS --> 10x compression +# HSMOOTH is the smoothing to apply to HCOMPRESSed data when decompressing; 0 for none + +# BITPIX(S32) is the bits per pixel for writing the output data +# COMPRESSION(STR) = NONE|RICE|GZIP|HCOMPRESS|PLIO is the compression algorithm +# TILE.[XYZ](S32) are the tile sizes. 0 means entire the dimension, so (0,1,1) forms tiles from rows +# NOISE(S32) [0..16] is the number of "noise bits" to preserve when quantising floating point data +# HSCALE(S32) is the scale factor for lossy compression with HCOMPRESS; 0 or 1 for none; 2*RMS --> 10x compression +# HSMOOTH(S32) is the smoothing to apply to HCOMPRESSed data when decompressing; 0 for none +# SCALING(STR) = NONE|RANGE|STDEV_POSITIVE|STDEV_NEGATIVE|STDEV_BOTH|MANUAL is the scaling scheme +# BSCALE(F32) is the manual scaling to apply (when SCALING = MANUAL) +# BZERO(F32) is the manual zero-point to apply (when SCALING = MANUAL) +# STDEV.BITS(S32) is the number of bits to map to a standard deviation (when SCALING = STDEV_*) +# STDEV.NUM(F32) is the number of standard deviations to the edge (when SCALING = STDEV_NEGATIVE|STDEV_POSITIVE) +# FLOAT(STR) is the name of a custom floating-point type + + DET_IMAGE METADATA + BITPIX S32 -32 + END + DET_MASK METADATA + BITPIX S32 8 + END + DET_WEIGHT METADATA + BITPIX S32 -32 + END + + SKY_IMAGE METADATA + BITPIX S32 -32 + END + SKY_MASK METADATA + BITPIX S32 8 + END + SKY_WEIGHT METADATA + BITPIX S32 -32 + END + + COMP_POS METADATA + BITPIX S32 16 + SCALING STR STDEV_POSITIVE + STDEV.BITS S32 4 + STDEV.NUM F32 10 + COMPRESSION STR RICE + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + COMP_MASK METADATA + BITPIX S32 8 + COMPRESSION STR PLIO + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + COMP_SUB METADATA + BITPIX S32 16 + SCALING STR STDEV_BOTH + STDEV.BITS S32 4 + STDEV.NUM F32 5 + COMPRESSION STR RICE + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + +END + +FILERULES METADATA + ### Redirections + PSASTRO.INPUT STR PSASTRO.INPUT.CMF + PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.MEF + PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF + PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.MEF + PPIMAGE.OUTPUT STR PPIMAGE.OUTPUT.MEF + PPIMAGE.BIN1 STR PPIMAGE.BIN1.MEF + PPIMAGE.BIN2 STR PPIMAGE.BIN2.MEF + DVOCORR.OUTPUT STR DVOCORR.MEF.OUTPUT + DVOFLAT.OUTPUT STR DVOFLAT.MEF.OUTPUT + PPIMAGE.OUTPUT.MASK STR PPIMAGE.OUT.MK.MEF + PPIMAGE.OUTPUT.WEIGHT STR PPIMAGE.OUT.WT.MEF + + ### input file definitions + ### use @DETDB entries to get the detrend images from the database + ### replace @DETDB with @FILES if you want to require it from the + ### command line, or with an explicit name to require a specific file + TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + + ## files used by ppImage + PPIMAGE.INPUT INPUT @FILES CHIP IMAGE + PPIMAGE.MASK INPUT @DETDB CHIP IMAGE + PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE + PPIMAGE.DARK INPUT @DETDB CHIP DARK + PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE + PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE + PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE + + ## files used to build and apply the flat-field correction images + DVOCORR.INPUT INPUT @FILES CHIP IMAGE + DVOCORR.REFHEAD INPUT @FILES CHIP HEADER + DVOFLAT.INPUT INPUT @FILES CHIP IMAGE + DVOFLAT.CORR INPUT @DETDB CHIP IMAGE + + ## files used by psphot + PSPHOT.LOAD INPUT @FILES CHIP IMAGE + PSPHOT.INPUT INPUT @FILES CHIP IMAGE + PSPHOT.MASK INPUT @FILES CHIP MASK + PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT + PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF + PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + + ## files used by psastro + PSASTRO.INPUT.CMP INPUT @FILES CHIP CMP + PSASTRO.INPUT.CMF INPUT @FILES CHIP CMF + + ## files used by pswarp + PSWARP.INPUT INPUT @FILES CHIP IMAGE + PSWARP.WEIGHT INPUT @FILES CHIP WEIGHT + PSWARP.MASK INPUT @FILES CHIP MASK + PSWARP.SKYCELL INPUT @FILES CHIP IMAGE + PSWARP.ASTROM INPUT @FILES CHIP CMF + + PPSUB.INPUT INPUT @FILES FPA IMAGE + PPSUB.INPUT.MASK INPUT @FILES FPA MASK + PPSUB.INPUT.WEIGHT INPUT @FILES FPA WEIGHT + PPSUB.REF INPUT @FILES FPA IMAGE + PPSUB.REF.MASK INPUT @FILES FPA MASK + PPSUB.REF.WEIGHT INPUT @FILES FPA WEIGHT + PPSUB.SOURCES INPUT @FILES FPA CMF + + PPSTACK.INPUT INPUT @FILES FPA IMAGE + PPSTACK.INPUT.MASK INPUT @FILES FPA MASK + PPSTACK.INPUT.WEIGHT INPUT @FILES FPA WEIGHT + PPSTACK.SOURCES INPUT @FILES FPA CMF + + PPSTAMP.INPUT INPUT @FILES CHIP IMAGE + + PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE + PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + + ### output file definitions + TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT + PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + PPIMAGE.OUTPUT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b0.fits IMAGE NONE CHIP TRUE SPLIT + PPIMAGE.OUTPUT.MEF OUTPUT {OUTPUT}.b0.fits IMAGE COMP_SUB CHIP TRUE MEF + #PPIMAGE.OUTPUT.DETMASK OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE COMP_MASK CHIP TRUE NONE + PPIMAGE.OUTPUT.DETMASK OUTPUT {OUTPUT}.detmask.fits IMAGE COMP_MASK CHIP TRUE MEF + + PPIMAGE.OUT.MK.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits MASK COMP_MASK CHIP TRUE SPLIT + PPIMAGE.OUT.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT COMP_POS CHIP TRUE SPLIT + PPIMAGE.OUT.MK.MEF OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK CHIP TRUE MEF + PPIMAGE.OUT.WT.MEF OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_POS CHIP TRUE MEF + + PPIMAGE.CHIP OUTPUT {OUTPUT}.fits IMAGE COMP_POS CHIP TRUE MEF + PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK CHIP TRUE MEF + PPIMAGE.CHIP.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_POS CHIP TRUE MEF + + PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE MEF + PPIMAGE.OUTPUT.FPA2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE MEF + + PPIMAGE.BIN1.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE NONE CHIP TRUE SPLIT + PPIMAGE.BIN2.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE NONE CHIP TRUE SPLIT + PPIMAGE.BIN1.MEF OUTPUT {OUTPUT}.b1c.fits IMAGE NONE CHIP TRUE MEF + PPIMAGE.BIN2.MEF OUTPUT {OUTPUT}.b2c.fits IMAGE NONE CHIP TRUE MEF + + PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE + PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE + + PPMERGE.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE MEF + + DVOCORR.MEF.OUTPUT OUTPUT {OUTPUT}.fc.fits IMAGE NONE CHIP TRUE MEF + DVOCORR.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE NONE CHIP TRUE SPLIT + DVOFLAT.MEF.OUTPUT OUTPUT {OUTPUT}.co.fits IMAGE NONE CHIP TRUE MEF + DVOFLAT.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE NONE CHIP TRUE SPLIT + + PSPHOT.RESID OUTPUT {OUTPUT}.res.fits IMAGE NONE CHIP TRUE MEF + PSPHOT.BACKGND OUTPUT {OUTPUT}.bck.fits IMAGE NONE CHIP TRUE MEF + PSPHOT.BACKSUB OUTPUT {OUTPUT}.sub.fits IMAGE NONE CHIP TRUE MEF + PSPHOT.BACKMDL OUTPUT {OUTPUT}.mdl.fits IMAGE NONE CHIP TRUE MEF + + PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW NONE CHIP TRUE NONE + PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.sx SX NONE CHIP TRUE NONE + PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.obj OBJ NONE CHIP TRUE NONE + PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.cmp CMP NONE CHIP TRUE NONE + PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF NONE CHIP TRUE NONE + PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE + + PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE CHIP TRUE MEF + + SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.mnt.png KAPA NONE FPA TRUE NONE + SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.psf.png KAPA NONE FPA TRUE NONE + SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.dap.png KAPA NONE FPA TRUE NONE + + PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE + PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE NONE + PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE NONE + + PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK NONE FPA TRUE NONE + PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE + PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + + SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + + PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK NONE FPA TRUE NONE + PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + + PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK NONE FPA TRUE NONE + PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + + PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + + PPSIM.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT + + PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE + PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE + + LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE + LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE + TRACE.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.trace TEXT NONE CHIP TRUE NONE +END + +EXTNAME.RULES METADATA + CMF.HEAD STR {CHIP.NAME}.hdr + CMF.DATA STR {CHIP.NAME}.psf # use .PSF and .EXT? + + PSF.HEAD STR {CHIP.NAME}.hdr + PSF.TABLE STR {CHIP.NAME}.psf_model + PSF.RESID STR {CHIP.NAME}.psf_resid +END + +BLANK.HEADERS METADATA + FPA.TIME STR MJD-OBS + FPA.EXPOSURE STR EXPTIME + FPA.AIRMASS STR AIRMASS +END Index: /tags/pap_tags/pap_root_080320/ippconfig/esowfi/filerules-mef.mdc =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/esowfi/filerules-mef.mdc (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/esowfi/filerules-mef.mdc (revision 22293) @@ -0,0 +1,130 @@ +### Redirections +PSASTRO.INPUT STR PSASTRO.INPUT.CMF +PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.SPL +PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF +PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.SPL + +### input file definitions +### use @DETDB entries to get the detrend images from the database +### replace @DETDB with @FILES if you want to require it from the +### command line, or with an explicit name to require a specific file +TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + +## files used by ppImage +PPIMAGE.INPUT INPUT @FILES CHIP IMAGE +PPIMAGE.MASK INPUT @DETDB CHIP IMAGE +PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE +PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE +PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE +PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE + +## files used to build and apply the flat field correction images +DVOCORR.INPUT INPUT @FILES CHIP IMAGE +DVOCORR.REFHEAD INPUT @FILES CHIP HEADER +DVOFLAT.INPUT INPUT @FILES CHIP IMAGE +DVOFLAT.CORR INPUT @DETDB CHIP IMAGE + +## files used by psphot +PSPHOT.LOAD INPUT @FILES CHIP IMAGE +PSPHOT.INPUT INPUT @FILES CHIP IMAGE +PSPHOT.MASK INPUT @FILES CHIP MASK +PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT +PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF +PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by psastro +PSASTRO.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by pswarp +PSWARP.INPUT INPUT @FILES CHIP IMAGE +PSWARP.SKYCELL INPUT @FILES CHIP IMAGE +PSWARP.WEIGHT INPUT @FILES CHIP WEIGHT +PSWARP.MASK INPUT @FILES CHIP MASK +PSWARP.ASTROM INPUT @FILES CHIP CMF + +PPSUB.INPUT INPUT none.fits CHIP IMAGE +PPSUB.INPUT.MASK INPUT none.fits CHIP MASK +PPSUB.INPUT.WEIGHT INPUT none.fits CHIP WEIGHT +PPSUB.REF INPUT none.fits CHIP IMAGE +PPSUB.REF.MASK INPUT none.fits CHIP MASK +PPSUB.REF.WEIGHT INPUT none.fits CHIP WEIGHT + +PPSTACK.INPUT INPUT none.fits CHIP IMAGE +PPSTACK.INPUT.MASK INPUT none.fits FPA MASK + +PPSTAMP.INPUT INPUT @FILES CHIP IMAGE + +PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE +PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + +### output file definitions +TYPE OUTPUT FILENAME.RULE FILE.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT +PPIMAGE.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE CHIP TRUE SPLIT +PPIMAGE.OUTPUT.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.mask.fits MASK CHIP TRUE SPLIT +PPIMAGE.OUTPUT.WEIGHT OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT CHIP TRUE SPLIT +PPIMAGE.CHIP OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE CHIP TRUE SPLIT +PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.ch.mask.fits MASK CHIP TRUE SPLIT +PPIMAGE.CHIP.WEIGHT OUTPUT {OUTPUT}.{CHIP.NAME}.ch.wt.fits WEIGHT CHIP TRUE SPLIT +PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.fpa1.fits IMAGE FPA TRUE TOGETHER +PPIMAGE.OUTPUT.FPA2 OUTPUT {OUTPUT}.fpa2.fits IMAGE FPA TRUE TOGETHER +PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS FPA TRUE TOGETHER + +PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG FPA TRUE TOGETHER +PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG FPA TRUE TOGETHER +PPIMAGE.BIN1 OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE CHIP TRUE SPLIT +PPIMAGE.BIN2 OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE CHIP TRUE SPLIT + +PPMERGE.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE FPA TRUE NONE + +DVOCORR.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE CHIP TRUE SPLIT +DVOFLAT.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE CHIP TRUE SPLIT + +PSPHOT.RESID OUTPUT {OUTPUT}.{CHIP.NAME}.res.fits IMAGE CHIP TRUE SPLIT +PSPHOT.BACKGND OUTPUT {OUTPUT}.{CHIP.NAME}.bck.fits IMAGE CHIP TRUE SPLIT +PSPHOT.BACKSUB OUTPUT {OUTPUT}.{CHIP.NAME}.sub.fits IMAGE CHIP TRUE SPLIT +PSPHOT.BACKMDL OUTPUT {OUTPUT}.{CHIP.NAME}.mdl.fits IMAGE CHIP TRUE SPLIT +PSPHOT.BACKMDL.STDEV OUTPUT {OUTPUT}.{CHIP.NAME}.mdd.fits IMAGE CHIP TRUE SPLIT + +PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW CHIP TRUE SPLIT +PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.{CHIP.NAME}.sx SX CHIP TRUE SPLIT +PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.{CHIP.NAME}.obj OBJ CHIP TRUE SPLIT +PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF CHIP TRUE SPLIT +PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF FPA TRUE TOGETHER + +PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.{CHIP.NAME}.psf PSF CHIP TRUE SPLIT + +SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.{CHIP.NAME}.mnt.png KAPA CHIP TRUE SPLIT +SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.{CHIP.NAME}.psf.png KAPA CHIP TRUE SPLIT +SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.{CHIP.NAME}.dap.png KAPA CHIP TRUE SPLIT + +PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP CHIP TRUE SPLIT +PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF CHIP TRUE SPLIT +PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF FPA TRUE TOGETHER + +PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE FPA TRUE TOGETHER +PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK FPA TRUE TOGETHER +PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT FPA TRUE TOGETHER +PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE FPA TRUE TOGETHER +PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE FPA TRUE TOGETHER + +SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS FPA TRUE NONE +SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL FPA TRUE NONE + +PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE FPA TRUE TOGETHER +PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK FPA TRUE TOGETHER +PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT FPA TRUE TOGETHER + +PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE FPA TRUE TOGETHER +PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK FPA TRUE TOGETHER + +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + +PPSIM.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE CHIP TRUE SPLIT + +PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE + +LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT CHIP TRUE SPLIT +LOG.EXP OUTPUT {OUTPUT}.log TEXT FPA TRUE TOGETHER Index: /tags/pap_tags/pap_root_080320/ippconfig/gpc1/filerules.mdc =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/gpc1/filerules.mdc (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/gpc1/filerules.mdc (revision 22293) @@ -0,0 +1,138 @@ +PSASTRO.INPUT STR PSASTRO.INPUT.CMF +PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.SPL +PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF +PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.SPL + +### input file definitions +TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE +PPIMAGE.INPUT INPUT none.fits CHIP IMAGE +INPUT.MASK INPUT none.fits CHIP MASK +INPUT.WEIGHT INPUT none.fits CHIP WEIGHT +INPUT.PSF INPUT none.fits CHIP PSF +INPUT.SRC INPUT none.fits CHIP CMF +PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE +PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE +PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE +PPIMAGE.MASK INPUT @DETDB CHIP MASK + +## files used by psphot +PSPHOT.LOAD INPUT @FILES CHIP IMAGE +PSPHOT.INPUT INPUT @FILES CHIP IMAGE +PSPHOT.MASK INPUT @FILES CHIP MASK +PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT +PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF +PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + +PSWARP.INPUT INPUT none.fits CHIP IMAGE +PSWARP.WEIGHT INPUT none.fits CHIP WEIGHT +PSWARP.MASK INPUT none.fits CHIP MASK +PSWARP.SKYCELL INPUT none.fits FPA IMAGE +PSWARP.ASTROM INPUT none.fits CHIP CMF + +PSASTRO.WCS INPUT none.fits CHIP CMF +PSASTRO.MODEL INPUT @DETDB FPA ASTROM + +PSASTRO.INPUT.CMP INPUT none.fits CHIP CMP +PSASTRO.INPUT.CMF INPUT none.fits CHIP CMF + +PPSUB.INPUT INPUT none.fits FPA IMAGE +PPSUB.INPUT.MASK INPUT none.fits FPA MASK +PPSUB.INPUT.WEIGHT INPUT none.fits FPA WEIGHT +PPSUB.REF INPUT none.fits FPA IMAGE +PPSUB.REF.MASK INPUT none.fits FPA MASK +PPSUB.REF.WEIGHT INPUT none.fits FPA WEIGHT + +PPSTACK.INPUT INPUT @FILES FPA IMAGE +PPSTACK.INPUT.MASK INPUT @FILES FPA MASK +PPSTACK.INPUT.WEIGHT INPUT @FILES FPA WEIGHT +PPSTACK.SOURCES INPUT @FILES FPA CMF + +PPSTAMP.INPUT INPUT none.fits CHIP IMAGE + +PPARITH.INPUT.IMAGE INPUT none.fits CHIP IMAGE +PPARITH.INPUT.MASK INPUT none.fits CHIP MASK + +### output file definitions +TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT +PPIMAGE.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE COMP_POS CHIP TRUE NONE +PPIMAGE.OUTPUT.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits MASK COMP_MASK CHIP TRUE NONE +PPIMAGE.OUTPUT.WEIGHT OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT COMP_POS CHIP TRUE NONE +PPIMAGE.OUTPUT.DETMASK OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE COMP_MASK CHIP TRUE NONE + +PPIMAGE.CHIP.RAW OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE COMP_POS CHIP TRUE NONE +# PPIMAGE.CHIP.MK OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE COMP_MASK CHIP TRUE NONE +PPIMAGE.CHIP OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE COMP_POS CHIP TRUE NONE +PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.ch.mk.fits MASK COMP_MASK CHIP TRUE NONE +PPIMAGE.CHIP.WEIGHT OUTPUT {OUTPUT}.{CHIP.NAME}.ch.wt.fits WEIGHT COMP_POS CHIP TRUE NONE + +PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.OUTPUT.FPA2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + +PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.BIN1 OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE NONE CHIP TRUE NONE +PPIMAGE.BIN2 OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE NONE CHIP TRUE NONE + +PPIMAGE.STATS OUTPUT {OUTPUT}.{CHIP.NAME}.stats STATS NONE CHIP TRUE NONE + +PPMERGE.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE + +DVOCORR.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE NONE CHIP TRUE NONE +DVOFLAT.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE NONE CHIP TRUE NONE + +PSPHOT.RESID OUTPUT {OUTPUT}.{CHIP.NAME}.res.fits IMAGE COMP_SUB CHIP TRUE NONE +PSPHOT.BACKGND OUTPUT {OUTPUT}.{CHIP.NAME}.bck.fits IMAGE COMP_POS CHIP TRUE NONE +PSPHOT.BACKSUB OUTPUT {OUTPUT}.{CHIP.NAME}.sub.fits IMAGE COMP_SUB CHIP TRUE NONE +PSPHOT.BACKMDL OUTPUT {OUTPUT}.{CHIP.NAME}.mdl.fits IMAGE COMP_POS CHIP TRUE NONE + +PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW NONE CHIP TRUE NONE +PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.{CHIP.NAME}.sx SX NONE CHIP TRUE NONE +PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.{CHIP.NAME}.obj OBJ NONE CHIP TRUE NONE +PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.cmp CMP NONE CHIP TRUE NONE +PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF NONE CHIP TRUE NONE +PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE MEF + +PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE CHIP TRUE MEF + +SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.{CHIP.NAME}.mnt.png KAPA NONE CHIP TRUE NONE +SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.{CHIP.NAME}.psf.png KAPA NONE CHIP TRUE NONE +SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.{CHIP.NAME}.dap.png KAPA NONE CHIP TRUE NONE + +PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE +PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE NONE +PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUT.MODEL OUTPUT {OUTPUT}.asm ASTROM.MODEL NONE FPA TRUE NONE +PSASTRO.OUT.REFSTARS OUTPUT {OUTPUT}.aref.fits ASTROM.REFSTARS NONE FPA TRUE NONE + +PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_POS FPA TRUE NONE +PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK COMP_MASK FPA TRUE NONE +PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_POS FPA TRUE NONE +PSWARP.OUTPUT.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE +PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE +PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + +SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE +SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + +PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_SUB FPA TRUE NONE +PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK COMP_MASK FPA TRUE NONE +PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_POS FPA TRUE NONE + +PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_POS FPA TRUE NONE +PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK COMP_MASK FPA TRUE NONE +PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_POS FPA TRUE NONE + +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.CHIP OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + +PPSIM.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE MEF + +PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE COMP_POS CHIP TRUE NONE +PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK COMP_MASK CHIP TRUE NONE + +LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE +LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE + +TRACE.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.trace TEXT NONE CHIP TRUE NONE +TRACE.EXP OUTPUT {OUTPUT}.trace TEXT NONE FPA TRUE NONE Index: /tags/pap_tags/pap_root_080320/ippconfig/isp/camera.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/isp/camera.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/isp/camera.config (revision 22293) @@ -0,0 +1,318 @@ +# Camera configuration file for the Pan-STARRS Imaging Sky Probe + +# File formats that we know about +FORMATS METADATA + ISP STR isp/format.config + CMP STR isp/cmp.config + CMF STR isp/cmf.config +END + +# Description of camera --- all the chips and the cells that comprise them +FPA METADATA + Chip STR Cell +END + +# valid filter names and corresponding IDs +FILTER.ID METADATA + g STR g + r STR r + i STR i + z STR z + y STR y +END + +# Table of strings to use for the class identifier (see ippdb) according to the file level. +CLASSID METADATA + FPA STR fpa +END + +DVO.CAMERADIR STR isp # Camera directory for DVO + +# convert supplied FPA.OBSTYPE values to abstract exptype names +OBSTYPE.TABLE METADATA + bias STR BIAS + zero STR BIAS + dark STR DARK + flat STR SKYFLAT + skyflat STR SKYFLAT + domeflat STR DOMEFLAT + object STR OBJECT + science STR OBJECT +END + +# Recipe options +RECIPES METADATA + PSPHOT STR isp/psphot.config # psphot details + PSASTRO STR isp/psastro.config # psastro details + PPIMAGE STR isp/ppImage.config # Recipes for ppImage + PPMERGE STR isp/ppMerge.config # Recipes for ppMerge + REJECTIONS STR isp/rejections.config +END + +# Reduction classes +REDUCTION METADATA + # Detrend processing + DETREND METADATA + BIAS_PROCESS STR PPIMAGE_O + BIAS_RESID STR PPIMAGE_B + BIAS_VERIFY STR PPIMAGE_OB + BIAS_STACK STR PPMERGE_BIAS + DARK_PROCESS STR PPIMAGE_OB + DARK_RESID STR PPIMAGE_D + DARK_VERIFY STR PPIMAGE_OBD + DARK_STACK STR PPMERGE_DARK + SHUTTER_PROCESS STR PPIMAGE_OBD + SHUTTER_RESID STR PPIMAGE_S + SHUTTER_VERIFY STR PPIMAGE_OBDS + SHUTTER_STACK STR PPMERGE_SHUTTER + FLAT_PROCESS STR PPIMAGE_OBDS + FLAT_RESID STR PPIMAGE_F + FLAT_VERIFY STR PPIMAGE_OBDSF + FLAT_STACK STR PPMERGE_FLAT + FRINGE_PROCESS STR PPIMAGE_OBDSF + FRINGE_RESID STR PPIMAGE_R + FRINGE_VERIFY STR PPIMAGE_OBDSFR + FRINGE_STACK STR PPMERGE_FRINGE + + JPEG_BIN1_IMAGE_BIAS STR PPIMAGE_J1_IMAGE_B + JPEG_BIN1_IMAGE_DARK STR PPIMAGE_J1_IMAGE_B + JPEG_BIN1_IMAGE_SHUTTER STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_FLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_DOMEFLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_SKYFLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_FRINGE STR PPIMAGE_J1_IMAGE_R + JPEG_BIN2_IMAGE_BIAS STR PPIMAGE_J2_IMAGE_B + JPEG_BIN2_IMAGE_DARK STR PPIMAGE_J2_IMAGE_B + JPEG_BIN2_IMAGE_SHUTTER STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_FLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_DOMEFLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_SKYFLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_FRINGE STR PPIMAGE_J2_IMAGE_R + + JPEG_BIN1_RESID_BIAS STR PPIMAGE_J1_RESID_B + JPEG_BIN1_RESID_DARK STR PPIMAGE_J1_RESID_B + JPEG_BIN1_RESID_SHUTTER STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_FLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_DOMEFLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_SKYFLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_FRINGE STR PPIMAGE_J1_RESID_R + JPEG_BIN2_RESID_BIAS STR PPIMAGE_J2_RESID_B + JPEG_BIN2_RESID_DARK STR PPIMAGE_J2_RESID_B + JPEG_BIN2_RESID_SHUTTER STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_FLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_DOMEFLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_SKYFLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_FRINGE STR PPIMAGE_J2_RESID_R + END + # Processing raw data + DEFAULT METADATA + CHIP STR PPIMAGE_OBDSFRA + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + END + + # Detrend Processing only for raw data + DETREND_ONLY METADATA + CHIP STR PPIMAGE_DET_ONLY + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + END +END + +FITS METADATA +# BITPIX is the bits per pixel for writing the output data +# COMP = NONE|RICE|GZIP|HCOMPRESS|PLIO is the compression algorithm +# TILE.[XYZ] are the tile sizes. 0 means entire the dimension, so (0,1,1) forms tiles from rows +# NOISE [0..16] is the number of "noise bits" to preserve when quantising floating point data; 16 for no loss +# HSCALE is the scale factor for lossy compression with HCOMPRESS; 0 or 1 for none; 2*RMS --> 10x compression +# HSMOOTH is the smoothing to apply to HCOMPRESSed data when decompressing; 0 for none + +# BITPIX(S32) is the bits per pixel for writing the output data +# COMPRESSION(STR) = NONE|RICE|GZIP|HCOMPRESS|PLIO is the compression algorithm +# TILE.[XYZ](S32) are the tile sizes. 0 means entire the dimension, so (0,1,1) forms tiles from rows +# NOISE(S32) [0..16] is the number of "noise bits" to preserve when quantising floating point data +# HSCALE(S32) is the scale factor for lossy compression with HCOMPRESS; 0 or 1 for none; 2*RMS --> 10x compression +# HSMOOTH(S32) is the smoothing to apply to HCOMPRESSed data when decompressing; 0 for none +# SCALING(STR) = NONE|RANGE|STDEV_POSITIVE|STDEV_NEGATIVE|STDEV_BOTH|MANUAL is the scaling scheme +# BSCALE(F32) is the manual scaling to apply (when SCALING = MANUAL) +# BZERO(F32) is the manual zero-point to apply (when SCALING = MANUAL) +# STDEV.BITS(S32) is the number of bits to map to a standard deviation (when SCALING = STDEV_*) +# STDEV.NUM(F32) is the number of standard deviations to the edge (when SCALING = STDEV_NEGATIVE|STDEV_POSITIVE) +# FLOAT(STR) is the name of a custom floating-point type + + DET_IMAGE METADATA + BITPIX S32 -32 + END + DET_MASK METADATA + BITPIX S32 8 + END + DET_WEIGHT METADATA + BITPIX S32 -32 + END + + SKY_IMAGE METADATA + BITPIX S32 -32 + END + SKY_MASK METADATA + BITPIX S32 8 + END + SKY_WEIGHT METADATA + BITPIX S32 -32 + END + + COMPRESSED_POSITIVE METADATA + BITPIX S32 16 + SCALING STR STDEV_POSITIVE + STDEV.BITS S32 4 + STDEV.NUM F32 10 + COMPRESSION STR RICE + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + COMPRESSED_MASK METADATA + COMPRESSION STR PLIO + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + COMPRESSED_SUBTRACTION METADATA + BITPIX S32 16 + SCALING STR STDEV_BOTH + STDEV.BITS S32 4 + STDEV.NUM F32 5 + COMPRESSION STR RICE + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + +END + +FILERULES METADATA + PSASTRO.INPUT STR PSASTRO.INPUT.CMF + PSASTRO.OUTPUT STR PSASTRO.OUTPUT.CMF + PSASTRO.OUTPUT.MEF STR PSASTRO.OUTPUT.CMF + PSPHOT.OUTPUT STR PSPHOT.OUTPUT.CMF + + ### input file definitions + ### use @DETDB entries to get the detrend images from the database + ### replace @DETDB with @FILES if you want to require it from the + ### command line, or with an explicit name to require a specific file + TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + + ## files used by ppImage + PPIMAGE.INPUT INPUT @FILES FPA IMAGE + PPIMAGE.MASK INPUT mask.fits FPA IMAGE + PPIMAGE.BIAS INPUT @DETDB FPA IMAGE + PPIMAGE.DARK INPUT @DETDB FPA DARK + PPIMAGE.FLAT INPUT @DETDB FPA IMAGE + PPIMAGE.FRINGE INPUT @DETDB FPA FRINGE + PPIMAGE.SHUTTER INPUT @DETDB FPA IMAGE + + ## files used to build and apply the flat-field correction images + DVOCORR.INPUT INPUT @FILES FPA IMAGE + DVOCORR.REFHEAD INPUT @FILES FPA HEADER + DVOFLAT.INPUT INPUT @FILES FPA IMAGE + DVOFLAT.CORR INPUT @DETDB FPA IMAGE + + ## files used by psphot + PSPHOT.LOAD INPUT @FILES FPA IMAGE + PSPHOT.INPUT INPUT @FILES FPA IMAGE + PSPHOT.MASK INPUT @FILES FPA MASK + PSPHOT.WEIGHT INPUT @FILES FPA WEIGHT + PSPHOT.PSF.LOAD INPUT @FILES FPA PSF + + ## files used by psastro + PSASTRO.INPUT.CMP INPUT @FILES FPA CMP + PSASTRO.INPUT.CMF INPUT @FILES FPA CMF + + ## files used by pswarp + PSWARP.INPUT INPUT @FILES FPA IMAGE + PSWARP.WEIGHT INPUT @FILES FPA WEIGHT + PSWARP.MASK INPUT @FILES FPA MASK + PSWARP.SKYCELL INPUT @FILES FPA IMAGE + PSWARP.ASTROM INPUT @FILES FPA CMF + + PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE + PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + + ### output file definitions + TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT + PPIMAGE.OUTPUT OUTPUT {OUTPUT}.isp.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE + PPIMAGE.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + PPIMAGE.CHIP OUTPUT {OUTPUT}.chip.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.chip.mask.fits MASK NONE FPA TRUE NONE + PPIMAGE.CHIP.WEIGHT OUTPUT {OUTPUT}.chip.wt.fits WEIGHT NONE FPA TRUE NONE + PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.OUTPUT.FPA2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + + PPIMAGE.BIN1 OUTPUT {OUTPUT}.b1c.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.BIN2 OUTPUT {OUTPUT}.b2c.fits IMAGE NONE FPA TRUE NONE + + PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE + PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE + + PPMERGE.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + + DVOCORR.OUTPUT OUTPUT {OUTPUT}.fc.fits IMAGE NONE FPA TRUE NONE + DVOFLAT.OUTPUT OUTPUT {OUTPUT}.co.fits IMAGE NONE FPA TRUE NONE + + PSPHOT.RESID OUTPUT {OUTPUT}.res.fits IMAGE NONE FPA TRUE NONE + PSPHOT.BACKGND OUTPUT {OUTPUT}.bck.fits IMAGE NONE FPA TRUE NONE + PSPHOT.BACKSUB OUTPUT {OUTPUT}.sub.fits IMAGE NONE FPA TRUE NONE + PSPHOT.BACKMDL OUTPUT {OUTPUT}.mdl.fits IMAGE NONE FPA TRUE NONE + + PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT} RAW NONE FPA TRUE NONE + PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.sx SX NONE FPA TRUE NONE + PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.obj OBJ NONE FPA TRUE NONE + PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.cmp CMP NONE FPA TRUE NONE + PSPHOT.OUTPUT.CMF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE + + PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE FPA TRUE NONE + + SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.mnt.png KAPA NONE FPA TRUE NONE + SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.psf.png KAPA NONE FPA TRUE NONE + SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.dap.png KAPA NONE FPA TRUE NONE + + PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.smp CMP NONE FPA TRUE NONE + PSASTRO.OUTPUT.CMF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE NONE + + PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE + PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE + PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + + SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + + PPSIM.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE ISP + PPSIM.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE + + PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE + PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE + + LOG.IMFILE OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE + LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE +END + +EXTNAME.RULES METADATA + CMF.HEAD STR cmf.hdr + CMF.DATA STR cmf.psf # use .PSF and .EXT? + + PSF.HEAD STR hdr + PSF.TABLE STR psf_model + PSF.RESID STR psf_resid +END + +BLANK.HEADERS METADATA + FPA.TIME STR MJD-OBS + FPA.EXPOSURE STR EXPTIME + FPA.AIRMASS STR AIRMASS +END Index: /tags/pap_tags/pap_root_080320/ippconfig/megacam/filerules-mef.mdc =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/megacam/filerules-mef.mdc (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/megacam/filerules-mef.mdc (revision 22293) @@ -0,0 +1,181 @@ +### File rules for PHU=FPA, EXT=CHIP + +### Redirections +PPIMAGE.OUTPUT STR PPIMAGE.OUTPUT.MEF +PPIMAGE.OUTPUT.MASK STR PPIMAGE.OUT.MK.MEF +PPIMAGE.OUTPUT.WEIGHT STR PPIMAGE.OUT.WT.MEF + +PPIMAGE.CHIP STR PPIMAGE.CHIP.MEF +PPIMAGE.CHIP.MASK STR PPIMAGE.CHIP.MK.MEF +PPIMAGE.CHIP.WEIGHT STR PPIMAGE.CHIP.WT.MEF + +PPIMAGE.OUTPUT.FPA1 STR PPIMAGE.OUTPUT.FPA1.MEF +PPIMAGE.OUTPUT.FPA2 STR PPIMAGE.OUTPUT.FPA2.MEF +PPIMAGE.BIN1 STR PPIMAGE.BIN1.MEF +PPIMAGE.BIN2 STR PPIMAGE.BIN2.MEF + +PPSTAMP.CHIP STR PPSTAMP.CHIP.MEF + +PSASTRO.INPUT STR PSASTRO.INPUT.CMF +PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.MEF +PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF +PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.MEF + +DVOCORR.OUTPUT STR DVOCORR.MEF.OUTPUT +DVOFLAT.OUTPUT STR DVOFLAT.MEF.OUTPUT + +### input file definitions +### use @DETDB entries to get the detrend images from the database +### replace @DETDB with @FILES if you want to require it from the +### command line, or with an explicit name to require a specific file +TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + +## files used by ppImage +PPIMAGE.INPUT INPUT @FILES CHIP IMAGE +PPIMAGE.MASK INPUT @DETDB CHIP MASK +PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE +PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE +PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE +PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE + +## files used to build and apply the flat-field correction images +DVOCORR.INPUT INPUT @FILES CHIP IMAGE +DVOCORR.REFHEAD INPUT @FILES CHIP HEADER +DVOFLAT.INPUT INPUT @FILES CHIP IMAGE +DVOFLAT.CORR INPUT @DETDB CHIP IMAGE + +## files used by psphot +PSPHOT.LOAD INPUT @FILES CHIP IMAGE +PSPHOT.INPUT INPUT @FILES CHIP IMAGE +PSPHOT.MASK INPUT @FILES CHIP MASK +PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT +PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF +PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by psastro +PSASTRO.WCS INPUT none.fits CHIP CMF +PSASTRO.MODEL INPUT @DETDB FPA ASTROM +PSASTRO.INPUT.CMP INPUT @FILES CHIP CMP +PSASTRO.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by pswarp +PSWARP.INPUT INPUT @FILES CHIP IMAGE +PSWARP.WEIGHT INPUT @FILES CHIP WEIGHT +PSWARP.MASK INPUT @FILES CHIP MASK +PSWARP.SKYCELL INPUT @FILES CHIP IMAGE +PSWARP.ASTROM INPUT @FILES CHIP CMF + +PPSUB.INPUT INPUT @FILES FPA IMAGE +PPSUB.INPUT.MASK INPUT @FILES FPA MASK +PPSUB.INPUT.WEIGHT INPUT @FILES FPA WEIGHT +PPSUB.REF INPUT @FILES FPA IMAGE +PPSUB.REF.MASK INPUT @FILES FPA MASK +PPSUB.REF.WEIGHT INPUT @FILES FPA WEIGHT +PPSUB.SOURCES INPUT @FILES FPA CMF + +PPSTACK.INPUT INPUT @FILES FPA IMAGE +PPSTACK.INPUT.MASK INPUT @FILES FPA MASK +PPSTACK.INPUT.WEIGHT INPUT @FILES FPA WEIGHT +PPSTACK.SOURCES INPUT @FILES FPA CMF + +PPSTAMP.INPUT INPUT @FILES CHIP IMAGE + +PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE +PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + +### output file definitions +TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT +PPIMAGE.OUTPUT.MEF OUTPUT {OUTPUT}.b0.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.OUT.MK.MEF OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK CHIP TRUE MEF +PPIMAGE.OUT.WT.MEF OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT CHIP TRUE MEF +PPIMAGE.OUTPUT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b0.fits IMAGE NONE CHIP TRUE SPLIT +PPIMAGE.OUT.MK.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits MASK NONE CHIP TRUE SPLIT +PPIMAGE.OUT.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT NONE CHIP TRUE SPLIT + +PPIMAGE.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.CHIP.MK.MEF OUTPUT {OUTPUT}.ch.mk.fits MASK COMP_MASK CHIP TRUE MEF +PPIMAGE.CHIP.WT.MEF OUTPUT {OUTPUT}.ch.wt.fits WEIGHT COMP_WT CHIP TRUE MEF +PPIMAGE.CHIP.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE NONE CHIP TRUE SPLIT +PPIMAGE.CHIP.MK.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.mk.fits MASK NONE CHIP TRUE SPLIT +PPIMAGE.CHIP.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.wt.fits WEIGHT NONE CHIP TRUE SPLIT + +PPIMAGE.OUTPUT.FPA1.MEF OUTPUT {OUTPUT}.b1.fits IMAGE COMP_IMG FPA TRUE MEF +PPIMAGE.OUTPUT.FPA2.MEF OUTPUT {OUTPUT}.b2.fits IMAGE COMP_IMG FPA TRUE MEF +PPIMAGE.OUTPUT.FPA1.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE NONE FPA TRUE SPLIT +PPIMAGE.OUTPUT.FPA2.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE NONE FPA TRUE SPLIT + +PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE MEF + +PPIMAGE.BIN1.MEF OUTPUT {OUTPUT}.b1c.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.BIN2.MEF OUTPUT {OUTPUT}.b2c.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.BIN1.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE COMP_IMG CHIP TRUE SPLIT +PPIMAGE.BIN2.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE COMP_IMG CHIP TRUE SPLIT + +PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE + +PPMERGE.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE MEF + +DVOCORR.MEF.OUTPUT OUTPUT {OUTPUT}.fc.fits IMAGE NONE CHIP TRUE MEF +DVOCORR.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE NONE CHIP TRUE SPLIT +DVOFLAT.MEF.OUTPUT OUTPUT {OUTPUT}.co.fits IMAGE NONE CHIP TRUE MEF +DVOFLAT.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE NONE CHIP TRUE SPLIT + +PSPHOT.RESID OUTPUT {OUTPUT}.res.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKGND OUTPUT {OUTPUT}.bck.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKSUB OUTPUT {OUTPUT}.sub.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKMDL OUTPUT {OUTPUT}.mdl.fits IMAGE NONE CHIP FALSE MEF + +PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW NONE CHIP TRUE NONE +PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.{CHIP.NAME}.sx SX NONE CHIP TRUE NONE +PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.{CHIP.NAME}.obj OBJ NONE CHIP TRUE NONE +PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.cmp CMP NONE CHIP TRUE NONE +PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF NONE CHIP TRUE SPLIT +PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF NONE CHIP TRUE MEF + +PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE CHIP TRUE MEF + +SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.mnt.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.psf.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.dap.png KAPA NONE FPA TRUE NONE + +PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE +PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE SPLIT +PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUT.MODEL OUTPUT {OUTPUT}.asm ASTROM.MODEL NONE FPA TRUE NONE +PSASTRO.OUT.REFSTARS OUTPUT {OUTPUT}.aref.fits ASTROM.REFSTARS NONE FPA TRUE NONE + +PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT FPA TRUE NONE +PSWARP.OUTPUT.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE +PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE COMP_IMG FPA TRUE NONE +PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE COMP_IMG FPA TRUE NONE + +SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE +SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + +PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT FPA TRUE NONE + +PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT FPA TRUE NONE + +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + +PPSIM.OUTPUT.MEF OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE MEF +PPSIM.OUTPUT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT +PPSIM.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE MEF + +PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE COMP_IMG CHIP TRUE NONE +PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK COMP_MASK CHIP TRUE NONE + +LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE +LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE + +TRACE.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.trace TEXT NONE CHIP TRUE NONE +TRACE.EXP OUTPUT {OUTPUT}.trace TEXT NONE FPA TRUE NONE Index: /tags/pap_tags/pap_root_080320/ippconfig/megacam/ppMerge.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/megacam/ppMerge.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/megacam/ppMerge.config (revision 22293) @@ -0,0 +1,40 @@ + +# Bias combination --- don't want min/max rejection +PPMERGE_BIAS METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 2 # Number of rejection iterations + FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately + FRACLOW F32 0.0 # Fraction of low pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR CLIPPED # Statistic to use for combination: +END + +# Dark combination --- don't want min/max rejection +# More aggressive clipping than bias, so as to remove CRs +PPMERGE_DARK METADATA + REJ F32 2.0 # Rejection threshold (sigma) + ITER S32 4 # Number of rejection iterations + FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately + FRACLOW F32 0.0 # Fraction of low pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR CLIPPED # Statistic to use for combination: +END + +# Flat combination --- use min/max rejection +PPMERGE_FLAT METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 1 # Number of rejection iterations + FRACHIGH F32 0.3 # Fraction of high pixels to reject immediately + FRACLOW F32 0.1 # Fraction of low pixels to reject immediately + NKEEP S32 5 # Minimum number of pixels in stack to keep + WEIGHTS BOOL TRUE # Use image weights? + COMBINE STR MEAN # Statistic to use for combination: +END + + +# Fringe combination --- already included in default, above +PPMERGE_FRINGE METADATA + FRACHIGH F32 0.1 # Fraction of high pixels to reject immediately + WEIGHTS BOOL TRUE # Use image weights? +END + Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/.cvsignore =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/.cvsignore (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/.cvsignore (revision 22293) @@ -0,0 +1,2 @@ +Makefile +Makefile.in Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/Makefile.am =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/Makefile.am (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/Makefile.am (revision 22293) @@ -0,0 +1,24 @@ + +installdir = $(datadir)/ippconfig/recipes + +install_files = \ + masks.config \ + rejections.config \ + ppImage.config \ + ppMerge.config \ + ppStack.config \ + ppStats.config \ + psastro.config \ + psphot.config \ + pswarp.config \ + ppSim.config \ + ppSub.config + +install_DATA = $(install_files) + +install-data-hook: + chmod 0755 $(installdir) + +EXTRA_DIST = $(install_files) + +ACLOCAL_AMFLAGS = -I m4 Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/filerules-mef.mdc =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/filerules-mef.mdc (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/filerules-mef.mdc (revision 22293) @@ -0,0 +1,178 @@ +### File rules for PHU=FPA, EXT=CHIP + +### Redirections +PPIMAGE.OUTPUT STR PPIMAGE.OUTPUT.MEF +PPIMAGE.OUTPUT.MASK STR PPIMAGE.OUT.MK.MEF +PPIMAGE.OUTPUT.WEIGHT STR PPIMAGE.OUT.WT.MEF + +PPIMAGE.CHIP STR PPIMAGE.CHIP.MEF +PPIMAGE.CHIP.MASK STR PPIMAGE.CHIP.MK.MEF +PPIMAGE.CHIP.WEIGHT STR PPIMAGE.CHIP.WT.MEF + +PPIMAGE.OUTPUT.FPA1 STR PPIMAGE.OUTPUT.FPA1.MEF +PPIMAGE.OUTPUT.FPA2 STR PPIMAGE.OUTPUT.FPA2.MEF +PPIMAGE.BIN1 STR PPIMAGE.BIN1.MEF +PPIMAGE.BIN2 STR PPIMAGE.BIN2.MEF + +PPSTAMP.CHIP STR PPSTAMP.CHIP.MEF + +PSASTRO.INPUT STR PSASTRO.INPUT.CMF +PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.MEF +PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF +PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.MEF + +DVOCORR.OUTPUT STR DVOCORR.MEF.OUTPUT +DVOFLAT.OUTPUT STR DVOFLAT.MEF.OUTPUT + +### input file definitions +### use @DETDB entries to get the detrend images from the database +### replace @DETDB with @FILES if you want to require it from the +### command line, or with an explicit name to require a specific file +TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + +## files used by ppImage +PPIMAGE.INPUT INPUT @FILES CHIP IMAGE +PPIMAGE.MASK INPUT @DETDB CHIP MASK +PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE +PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE +PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE +PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE + +## files used to build and apply the flat-field correction images +DVOCORR.INPUT INPUT @FILES CHIP IMAGE +DVOCORR.REFHEAD INPUT @FILES CHIP HEADER +DVOFLAT.INPUT INPUT @FILES CHIP IMAGE +DVOFLAT.CORR INPUT @DETDB CHIP IMAGE + +## files used by psphot +PSPHOT.LOAD INPUT @FILES CHIP IMAGE +PSPHOT.INPUT INPUT @FILES CHIP IMAGE +PSPHOT.MASK INPUT @FILES CHIP MASK +PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT +PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF +PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by psastro +PSASTRO.INPUT.CMP INPUT @FILES CHIP CMP +PSASTRO.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by pswarp +PSWARP.INPUT INPUT @FILES CHIP IMAGE +PSWARP.WEIGHT INPUT @FILES CHIP WEIGHT +PSWARP.MASK INPUT @FILES CHIP MASK +PSWARP.SKYCELL INPUT @FILES CHIP IMAGE +PSWARP.ASTROM INPUT @FILES CHIP CMF + +PPSUB.INPUT INPUT @FILES FPA IMAGE +PPSUB.INPUT.MASK INPUT @FILES FPA MASK +PPSUB.INPUT.WEIGHT INPUT @FILES FPA WEIGHT +PPSUB.REF INPUT @FILES FPA IMAGE +PPSUB.REF.MASK INPUT @FILES FPA MASK +PPSUB.REF.WEIGHT INPUT @FILES FPA WEIGHT +PPSUB.SOURCES INPUT @FILES FPA CMF + +PPSTACK.INPUT INPUT @FILES FPA IMAGE +PPSTACK.INPUT.MASK INPUT @FILES FPA MASK +PPSTACK.INPUT.WEIGHT INPUT @FILES FPA WEIGHT +PPSTACK.SOURCES INPUT @FILES FPA CMF + +PPSTAMP.INPUT INPUT @FILES CHIP IMAGE + +PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE +PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + +### output file definitions +TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT +PPIMAGE.OUTPUT.MEF OUTPUT {OUTPUT}.b0.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.OUT.MK.MEF OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK CHIP TRUE MEF +PPIMAGE.OUT.WT.MEF OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT CHIP TRUE MEF +PPIMAGE.OUTPUT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b0.fits IMAGE NONE CHIP TRUE SPLIT +PPIMAGE.OUT.MK.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits MASK NONE CHIP TRUE SPLIT +PPIMAGE.OUT.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT NONE CHIP TRUE SPLIT + +PPIMAGE.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.CHIP.MK.MEF OUTPUT {OUTPUT}.ch.mk.fits MASK COMP_MASK CHIP TRUE MEF +PPIMAGE.CHIP.WT.MEF OUTPUT {OUTPUT}.ch.wt.fits WEIGHT COMP_WT CHIP TRUE MEF +PPIMAGE.CHIP.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE NONE CHIP TRUE SPLIT +PPIMAGE.CHIP.MK.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.mk.fits MASK NONE CHIP TRUE SPLIT +PPIMAGE.CHIP.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.wt.fits WEIGHT NONE CHIP TRUE SPLIT + +PPIMAGE.OUTPUT.FPA1.MEF OUTPUT {OUTPUT}.b1.fits IMAGE COMP_IMG FPA TRUE MEF +PPIMAGE.OUTPUT.FPA2.MEF OUTPUT {OUTPUT}.b2.fits IMAGE COMP_IMG FPA TRUE MEF +PPIMAGE.OUTPUT.FPA1.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE NONE FPA TRUE SPLIT +PPIMAGE.OUTPUT.FPA2.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE NONE FPA TRUE SPLIT + +PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE MEF + +PPIMAGE.BIN1.MEF OUTPUT {OUTPUT}.b1c.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.BIN2.MEF OUTPUT {OUTPUT}.b2c.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.BIN1.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE COMP_IMG CHIP TRUE SPLIT +PPIMAGE.BIN2.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE COMP_IMG CHIP TRUE SPLIT + +PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE + +PPMERGE.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE MEF + +DVOCORR.MEF.OUTPUT OUTPUT {OUTPUT}.fc.fits IMAGE NONE CHIP TRUE MEF +DVOCORR.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE NONE CHIP TRUE SPLIT +DVOFLAT.MEF.OUTPUT OUTPUT {OUTPUT}.co.fits IMAGE NONE CHIP TRUE MEF +DVOFLAT.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE NONE CHIP TRUE SPLIT + +PSPHOT.RESID OUTPUT {OUTPUT}.res.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKGND OUTPUT {OUTPUT}.bck.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKSUB OUTPUT {OUTPUT}.sub.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKMDL OUTPUT {OUTPUT}.mdl.fits IMAGE NONE CHIP FALSE MEF + +PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW NONE CHIP TRUE NONE +PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.{CHIP.NAME}.sx SX NONE CHIP TRUE NONE +PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.{CHIP.NAME}.obj OBJ NONE CHIP TRUE NONE +PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.cmp CMP NONE CHIP TRUE NONE +PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF NONE CHIP TRUE SPLIT +PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF NONE CHIP TRUE MEF + +PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE CHIP TRUE MEF + +SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.mnt.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.psf.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.dap.png KAPA NONE FPA TRUE NONE + +PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE +PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE SPLIT +PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUT.ASTROM OUTPUT {OUTPUT}.ast ASTROM NONE FPA TRUE NONE + +PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT FPA TRUE NONE +PSWARP.OUTPUT.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE +PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE COMP_IMG FPA TRUE NONE +PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE COMP_IMG FPA TRUE NONE + +SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE +SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + +PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT FPA TRUE NONE + +PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT FPA TRUE NONE + +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + +PPSIM.OUTPUT.MEF OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE MEF +PPSIM.OUTPUT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT +PPSIM.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE MEF + +PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE COMP_IMG CHIP TRUE NONE +PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK COMP_MASK CHIP TRUE NONE + +LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE +LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE + +TRACE.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.trace TEXT NONE CHIP TRUE NONE +TRACE.EXP OUTPUT {OUTPUT}.trace TEXT NONE FPA TRUE NONE Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/masks.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/masks.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/masks.config (revision 22293) @@ -0,0 +1,10 @@ +### Recipe specifying values for various mask concepts +BLANK U8 0x01 # The pixel is blank or has no (valid) data +FLAT U8 0x02 # The pixel is non-positive in the flat-field +DETECTOR U8 0x02 # The detector pixel is bad (e.g., bad column, charge trap) +SAT U8 0x04 # The pixel is saturated in the image of interest +BAD U8 0x04 # The pixel is low in the image of interest +RANGE U8 0x04 # The pixel is out of range in the image of interest +CR U8 0x08 # The pixel is probably a CR +SUSPECT U8 0x40 # The pixel is suspected of being bad, but may not be +MARK U8 0x80 # The pixel is marked as temporarily ignored Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppImage.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppImage.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppImage.config (revision 22293) @@ -0,0 +1,1190 @@ +### ppImage recipe configuration file +### Here we have turned on our 'best guess' defaults for a simple camera +### Camera-specific options may be turned on by recipes in the camera configuration + +# List of tasks to perform +NONLIN BOOL FALSE # Non-linearity correction; not implemented +OVERSCAN BOOL TRUE # Overscan subtraction +BIAS BOOL TRUE # Bias subtraction +DARK BOOL TRUE # Dark subtraction +SHUTTER BOOL FALSE # Shutter correction +FLAT BOOL TRUE # Flat-field normalisation +MASK BOOL FALSE # Mask bad pixels +MASK.VALUE STR SAT,BAD # Mask pixels with these attributes +MASK.BUILD BOOL FALSE # Build internal mask image +REPLACE.MASKED BOOL FALSE # Fill in masked pixels +WEIGHT.BUILD BOOL FALSE # Build internal weight image +FRINGE BOOL FALSE # Fringe subtraction +PHOTOM BOOL FALSE # Source identification and photometry +ASTROM.CHIP BOOL FALSE # Astrometry per chip? +ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + +# output data formats to save +BASE.FITS BOOL FALSE # Save base detrended image? +BASE.MASK.FITS BOOL FALSE # Save base detrended image? +BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? +CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? +CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? +CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? +FPA1.FITS BOOL FALSE # Save 1st binned fpa image? +FPA2.FITS BOOL FALSE # Save 2nd binned fpa image? +BIN1.FITS BOOL FALSE # Save 1st binned chip image? +BIN2.FITS BOOL FALSE # Save 2nd binned chip image? +BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? +BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + +# Non-linearity correction +NONLIN.SOURCE STR CHIP.NAME # How to determine the source +NONLIN.DATA STR nonlin.dat # Filename for lookup table + +OLDDARK BOOL FALSE # Use old-style darks? + + +# examples of other possible non-linearity correction representations +#@NONLIN.DATA F32 0.0 1.001 0.001 # A polynomial +#NONLIN.DATA METADATA # Source of non-linearity data +# ccd00 STR nonlin00.dat # A lookup table +# @ccd01 F32 0.0 1.001 0.001 # A polynomial +#END + +# Overscan subtraction +OVERSCAN.SINGLE BOOL TRUE # Reduce overscan to a single value? +OVERSCAN.FIT STR NONE # NONE | POLYNOMIAL | SPLINE +OVERSCAN.ORDER S32 5 # Order of polynomial fit +OVERSCAN.STAT STR MEAN # MEAN | MEDIAN +OVERSCAN.BOXCAR S32 0 # Boxcar smoothing radius +OVERSCAN.GAUSS F32 0.0 # Gaussian smoothing sigma +OVERSCAN.CONSTANT BOOL FALSE # Apply a known, fixed value? +OVERSCAN.VALUE F32 0.0 # value to apply, if requested + +# Fringe subtraction options +FRINGE.ITER S32 10 # Number of rejection iterations for fringe solution +FRINGE.REJ F32 2.0 # Rejection threshold for fringe solution +FRINGE.KEEP F32 0.5 # Minimum fraction to keep in fringe solution + +# binned output image options +BIN1.XBIN S32 4 +BIN1.YBIN S32 4 +BIN2.XBIN S32 16 +BIN2.YBIN S32 16 + +PPIMAGE.JPEG1 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -5.0 + SCALE.MAX F32 +10.0 +END + +PPIMAGE.JPEG2 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR RANGE + SCALE.MIN STR -5.0 + SCALE.MAX STR +10.0 +END + +PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID}.{CHIP.N} + +FRINGE.FILTERS MULTI +FRINGE.FILTERS STR UNDEF + +DETREND.CONSTRAINTS METADATA +END + +################################################################################ +# Diffferent processing options, which may be loaded symbolically +################################################################################ + +# No operation except potential normalisation +PPIMAGE_N METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan subtraction only +PPIMAGE_O METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Bias subtraction only +PPIMAGE_B METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Dark subtraction only +PPIMAGE_D METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Shutter correction only +PPIMAGE_S METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL TRUE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Flat-fielding only +PPIMAGE_F METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Fringe correction only +PPIMAGE_R METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Photometry only +PPIMAGE_P METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Photometry & Astrometry +PPIMAGE_A METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL TRUE # Astromtery per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Overscan, bias +PPIMAGE_OB METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark +PPIMAGE_OBD METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark, shutter +PPIMAGE_OBDS METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark, shutter, flat-field +PPIMAGE_OBDSF METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark, shutter, flat-field, fringe +PPIMAGE_OBDSFR METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom, astrom +PPIMAGE_DET_ONLY METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom +PPIMAGE_OBDSFRP METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom, astrom +PPIMAGE_OBDSFRA METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL TRUE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Overscan, bias, dark, shutter, flat-field, fringe, photom, astrom +PPIMAGE_MASKPHOT METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL TRUE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Save JPEG from BIN1 +PPIMAGE_JPEG METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? +END + +# Save JPEG from BIN1 +PPIMAGE_J1 METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned +END + +# Save JPEG from BIN2 +PPIMAGE_J2 METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned +END + +# Overscan, bias, photometry, astrometry (for, eg, summit ISP analysis) +PPIMAGE_OA METADATA + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL TRUE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Overscan, bias, photometry (for quick analysis) +PPIMAGE_OP METADATA + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# Photometry and astrometry only (for pre-reduced data) +PPIMAGE_PA METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL TRUE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? +END + +# JPEG images for different types of residual images +# Save JPEG from BIN1 for bias +PPIMAGE_J1_RESID_B METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG1 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR VALUE + SCALE.MIN F32 -5.0 + SCALE.MAX F32 +5.0 + END +END + +# Save JPEG from BIN2 for bias +PPIMAGE_J2_RESID_B METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG2 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR VALUE + SCALE.MIN F32 -5.0 + SCALE.MAX F32 +5.0 + END +END + +# Save JPEG from BIN1 for flat +PPIMAGE_J1_RESID_F METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG1 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR FRACTION + SCALE.MIN F32 0.95 + SCALE.MAX F32 1.05 + END +END + +# Save JPEG from BIN2 for flat +PPIMAGE_J2_RESID_F METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG2 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR FRACTION + SCALE.MIN F32 0.95 + SCALE.MAX F32 1.05 + END +END + +# Save JPEG from BIN1 for flat +PPIMAGE_J1_RESID_R METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG1 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -2.0 + SCALE.MAX F32 +3.0 + END +END + +# Save JPEG from BIN2 for flat +PPIMAGE_J2_RESID_R METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG2 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -2.0 + SCALE.MAX F32 +3.0 + END +END + +# JPEG images for different types of positive images +# Save JPEG from BIN1 for bias +PPIMAGE_J1_IMAGE_B METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG1 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -5.0 + SCALE.MAX F32 +5.0 + END +END + +# Save JPEG from BIN2 for bias +PPIMAGE_J2_IMAGE_B METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG2 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -5.0 + SCALE.MAX F32 +5.0 + END +END + +# Save JPEG from BIN1 for flat +PPIMAGE_J1_IMAGE_F METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG1 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -5.0 + SCALE.MAX F32 +5.0 + END +END + +# Save JPEG from BIN2 for flat +PPIMAGE_J2_IMAGE_F METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG2 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -5.0 + SCALE.MAX F32 +5.0 + END +END + +# Save JPEG from BIN1 for flat +PPIMAGE_J1_IMAGE_R METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG1 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -2.0 + SCALE.MAX F32 +3.0 + END +END + +# Save JPEG from BIN2 for flat +PPIMAGE_J2_IMAGE_R METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG2 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -2.0 + SCALE.MAX F32 +3.0 + END +END + +# Save JPEG from BIN1 for mask +PPIMAGE_J1_IMAGE_M METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG1 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR FRACTION + SCALE.MIN F32 0.95 + SCALE.MAX F32 1.05 + END +END + +# Save JPEG from BIN2 for mask +PPIMAGE_J2_IMAGE_M METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG2 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR FRACTION + SCALE.MIN F32 0.95 + SCALE.MAX F32 1.05 + END +END + +# Save JPEG from BIN1 for mask +PPIMAGE_J1_RESID_M METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG1 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR FRACTION + SCALE.MIN F32 0.95 + SCALE.MAX F32 1.05 + END +END + +# Save JPEG from BIN2 for mask +PPIMAGE_J2_RESID_M METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.WEIGHT.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.WEIGHT.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + + PPIMAGE.JPEG2 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR FRACTION + SCALE.MIN F32 0.95 + SCALE.MAX F32 1.05 + END +END + Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppMerge.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppMerge.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppMerge.config (revision 22293) @@ -0,0 +1,89 @@ +# Recipe configuration for ppMerge + +ROWS S32 128 # Number of rows to read at once +ELECTRONS F32 100.0 # Minimum number of electrons for useful signal +SAMPLE S32 100 # Sampling factor for measuring the background +REJ F32 3.0 # Rejection threshold (sigma) +ITER S32 0 # Number of rejection iterations +FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately +FRACLOW F32 0.0 # Fraction of low pixels to reject immediately +NKEEP S32 5 # Minimum number of pixels in stack to keep +WEIGHTS BOOL FALSE # Use image weights in combination? +FRINGE.NUM S32 10000 # Number of fringe regions +FRINGE.SIZE S32 5 # Half-size of fringe regions +FRINGE.XSMOOTH S32 5 # Number of smoothing regions in x +FRINGE.YSMOOTH S32 11 # Number of smoothing regions in y +SHUTTER.SIZE S32 128 # Size for shutter measurement regions +SHUTTER.ITER S32 1 # Number of iterations for shutter measurement +SHUTTER.REJECT F32 2 # Rejection limit for shutter measurement +MASK.SUSPECT F32 5.0 # Threshold for suspect pixels (sigma) +MASK.BAD F32 4.0 # Threshold for bad pixels (sigma) +MASK.MODE STR POISSON # Threshold for bad pixels (sigma) +STATS.BY.CHIP BOOL TRUE # measure stats for masking by chip (or by readout) +MASK.GROW.NPIX S32 3 # measure stats for masking by chip (or by readout) +MASKVAL STR SAT,BAD # Mask value for input data +COMBINE STR CLIPPED # Statistic to use for combination +MEAN STR ROBUST_MEDIAN # Statistic to use to measure the mean +STDEV STR ROBUST_STDEV # Statistic to use to measure the stdev + +# Ordinates for fitting dark current +DARK.ORDINATES METADATA + CELL.DARKTIME S32 1 # Traditional dark current term +END +DARK.NORM STR NONE # Dark normalisation concept + +# Bias combination --- don't want min/max rejection +PPMERGE_BIAS METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 2 # Number of rejection iterations + FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately + FRACLOW F32 0.0 # Fraction of low pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR CLIPPED # Statistic to use for combination: +END + + +# Dark combination --- don't want min/max rejection +# More aggressive clipping than bias, so as to remove CRs +PPMERGE_DARK METADATA + REJ F32 2.0 # Rejection threshold (sigma) + ITER S32 4 # Number of rejection iterations + FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately + FRACLOW F32 0.0 # Fraction of low pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR CLIPPED # Statistic to use for combination: +END + +# Flat combination --- use min/max rejection +PPMERGE_FLAT METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 1 # Number of rejection iterations + FRACHIGH F32 0.3 # Fraction of high pixels to reject immediately + FRACLOW F32 0.1 # Fraction of low pixels to reject immediately + NKEEP S32 5 # Minimum number of pixels in stack to keep + WEIGHTS BOOL TRUE # Use image weights? + COMBINE STR MEAN # Statistic to use for combination: +END + + +# Fringe combination --- already included in default, above +PPMERGE_FRINGE METADATA + FRACHIGH F32 0.1 # Fraction of high pixels to reject immediately + WEIGHTS BOOL TRUE # Use image weights? +END + +# Mask generation --- already included in default, above +PPMERGE_DARKMASK METADATA + MASK.BAD F32 3.0 # Threshold for bad pixels (sigma) + MASK.MODE STR VALUE # Threshold for bad pixels (sigma) +END + +# Mask generation --- already included in default, above +PPMERGE_FLATMASK METADATA + MASK.BAD F32 3.0 # Threshold for bad pixels (sigma) + MASK.MODE STR VALUE # Threshold for bad pixels (sigma) +END + +# Shutter generation --- already included in default, above +PPMERGE_SHUTTER METADATA +END Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppSim.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppSim.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppSim.config (revision 22293) @@ -0,0 +1,66 @@ +### Recipe values for ppSim, the Pan-STARRS data simulator + +BIAS.LEVEL F32 1000.0 # Bias level (e) +BIAS.RANGE F32 20.0 # Range of bias values (e) +BIAS.ORDER S32 7 # Order for overscan polynomial +DARK.RATE F32 5.0 # Dark current (e/s) +FLAT.SIGMA F32 3.0 # Flat gaussian width (FPA sizes) +FLAT.RATE F32 2000.0 # Flat illumination rate (e/s) +SHUTTER.TIME F32 1.0 # Time for shutter to fully open +SKY.RATE F32 10.0 # Sky illumination rate (e/s) + +STARS.REAL BOOL TRUE # Add stars from a catalogue? + +STARS.FAKE BOOL TRUE # Add fake stars, randomly distributed? +STARS.LUM F32 -1.5 # Stellar luminosity function slope +STARS.MAG F32 15.5 # Brightest magnitude for fake stars +STARS.DENSITY F32 1.0 # Stellar density (per square degree) at the brightest magnitude + +GALAXY.FAKE BOOL FALSE # Generate fake galaxies? + +GALAXY.LUM F32 -0.8 # Stellar luminosity function slope +GALAXY.MAG F32 11.0 # Brightest magnitude for fake stars +GALAXY.DENSITY F32 6.0 # Galaxy density (per square degree) at the brightest magnitude + +GALAXY.RMAJOR.MIN F32 2.0 # Minimum semi-major axis for galaxy models +GALAXY.RMAJOR.MAX F32 10.0 # Maximum semi-major axis for galaxy models + +GALAXY.ARATIO.MIN F32 1.0 # Minimum axis ratio for galaxy models +GALAXY.ARATIO.MAX F32 0.2 # Maximum axis ratio for galaxy models + +GALAXY.INDEX.MIN F32 0.25 # Minimum Sersic index for galaxy models +GALAXY.INDEX.MAX F32 0.500 # Maximum Sersic index for galaxy models + +GALAXY.THETA.MIN F32 0.0 # Minimum position angle for galaxy models +GALAXY.THETA.MAX F32 3.14 # Maximum position angle for galaxy models + +GALAXY.GRID BOOL FALSE # Generate a (regular) grid of galaxy models? +GALAXY.GRID.DX S32 200 # Spacing between galaxies in grid in x +GALAXY.GRID.DY S32 200 # Spacing between galaxies in grid in y + +BADPIX.SEED U64 123456789 # Seed for RNG in creating deterministic bad pixels +BADPIX.FRAC F32 0.0001 # Fraction of bad pixels + +### The following options are used if not defined by the concepts (e.g., usually read from headers) +GAIN F32 1.0 # Default gain (e/ADU) +READNOISE F32 10.0 # Default read noise (e) +OVERSCAN.SIZE S32 32 # Default overscan columns +SATURATION F32 65535 # Default saturation level (ADU) + +PIXEL.SCALE F32 0.26 # pixel size in arcsec + +# filter-dependent parameters +ZEROPTS MULTI + +ZEROPTS METADATA + FILTER STR g + ZERO_PT F32 24.0 +END +ZEROPTS METADATA + FILTER STR r + ZERO_PT F32 25.15 +END +ZEROPTS METADATA + FILTER STR i + ZERO_PT F32 25.00 +END Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppStack.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppStack.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppStack.config (revision 22293) @@ -0,0 +1,21 @@ +# Recipe configuration for ppStack (image combination) + +ITER S32 1 # Number of rejection iterations +COMBINE.REJ F32 4.0 # Rejection threshold in combination (sigma) +MASK.BAD STR BLANK,SAT,BAD # Mask value of bad pixels +MASK.BLANK STR BLANK # Mask value to give blank pixels +THRESHOLD.MASK F32 0.8 # Threshold for mask deconvolution (0..1) +IMAGE.REJ F32 0.2 # Rejected pixel fraction threshold for rejecting entire image +ROWS S32 64 # Number of rows to read at once +VARIANCE BOOL TRUE # Use variance in rejection? +SAFE BOOL TRUE # Play safe when combining small number of values? + +PSF.INSTANCES S32 5 # Number of instances for PSF generation +PSF.RADIUS F32 20.0 # Radius for PSF generation +PSF.ORDER S32 3 # Order of spatial variation for PSF generation +PSF.MODEL STR PS_MODEL_RGAUSS # Model for PSF generation + +TEMP.IMAGE STR conv.im.fits # Suffix for convolved images +TEMP.MASK STR conv.mk.fits # Suffix for convolved masks +TEMP.WEIGHT STR conv.wt.fits # Suffix for convolved weight maps +TEMP.DELETE BOOL FALSE # Delete temporary files on completion? Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppStats.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppStats.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppStats.config (revision 22293) @@ -0,0 +1,129 @@ +### Default ppStats recipe + +# Options governing statistics +SAMPLE F32 0.1 # Fraction of cell to sample +MASKVAL STR SAT,BAD # Mask value to use for statistics + +# Define the outputs as MULTI +HEADER MULTI +CONCEPT MULTI +STAT MULTI +SUMMARY MULTI + +# Add dummy values, to define the MULTI --- due to limitations in the MDC language +HEADER STR _UNDEF +CONCEPT STR _UNDEF +STAT STR _UNDEF +SUMMARY STR _UNDEF +ANALYSIS STR _UNDEF + +# basic stats for output images +CHIPSTATS METADATA + STAT MULTI + STAT STR ROBUST_MEDIAN + STAT STR ROBUST_STDEV # Background statistics estimators + + HEADER MULTI + HEADER STR OVER_VAL + HEADER STR FWHM_X # major axis FWHM (in pixels) + HEADER STR FWHM_Y # minor axis FWHM (in pixels) + HEADER STR DT_DET # elapsed time in detrend processing + HEADER STR DT_PHOT # elapsed time in photometry processing + HEADER STR DT_ASTR # elapsed time in astrometry processing + HEADER STR APMIFIT + HEADER STR DAPMIFIT + HEADER STR CERROR + HEADER STR NSTARS + HEADER STR NASTRO + + ANALYSIS MULTI # metadata blocks to search in chip/cell/readout->analysis + ANALYSIS STR PSPHOT.HEADER + ANALYSIS STR PSASTRO.HEADER +END + +# basic stats for residual images +RESIDUAL METADATA + STAT MULTI + STAT STR ROBUST_MEDIAN + STAT STR ROBUST_STDEV # Background statistics estimators + STAT STR SAMPLE_SKEWNESS + STAT STR SAMPLE_KURTOSIS + HEADER MULTI + HEADER STR FRNG_00 # fringe amplitude, if measured + HEADER STR FRNG_00D # fringe error, if measured + HEADER STR FRNG_01 # fringe amplitude, if measured + HEADER STR FRNG_01D # fringe error, if measured + HEADER STR FRNG_02 # fringe amplitude, if measured + HEADER STR FRNG_02D # fringe error, if measured +END + +### ppStats recipe for injection +INJECT METADATA + CONCEPT MULTI +END + +### ppStats recipe for registration +REGISTER METADATA + CONCEPT MULTI + STAT MULTI + SUMMARY MULTI + + CONCEPT STR FPA.OBJECT # Object name + CONCEPT STR FPA.OBSTYPE # Observation type + CONCEPT STR FPA.FILTER # Filter + CONCEPT STR FPA.RA FPA.DEC # Telescope pointing + + CONCEPT STR FPA.COMMENT # Obs Comment + + CONCEPT STR FPA.AIRMASS # Airmass + CONCEPT STR FPA.ALT FPA.AZ # Telescopy alt/az + CONCEPT STR FPA.POSANGLE # Rotator angle + CONCEPT STR FPA.TIME # Time of exposure + CONCEPT STR FPA.TELESCOPE # Telescope (eg, CFHT) + CONCEPT STR FPA.INSTRUMENT # Instrument (eg, CFH12K) + CONCEPT STR CHIP.TEMP # Detector temperature + CONCEPT STR CELL.EXPOSURE # Exposure time + + CONCEPT STR FPA.M1X # Primary x position + CONCEPT STR FPA.M1Y # Primary y position + CONCEPT STR FPA.M1Z # Primary z position + CONCEPT STR FPA.M1TIP # Primary tip + CONCEPT STR FPA.M1TILT # Primary tilt + CONCEPT STR FPA.M2X # Secondary x position + CONCEPT STR FPA.M2Y # Secondary y position + CONCEPT STR FPA.M2Z # Secondary z position + CONCEPT STR FPA.M2TIP # Secondary tip position + CONCEPT STR FPA.M2TILT # Secondary tilt position + CONCEPT STR FPA.ENV.TEMP # external temperature + CONCEPT STR FPA.ENV.HUMID # external humidity + CONCEPT STR FPA.ENV.WIND # external wind speed + CONCEPT STR FPA.ENV.DIR # external wind direction + + CONCEPT STR FPA.TELTEMP.M1 # Primary mirror temps (C) + CONCEPT STR FPA.TELTEMP.M1CELL # Primary mirror support temps (C) + CONCEPT STR FPA.TELTEMP.M2 # Secondary mirror temps (C + CONCEPT STR FPA.TELTEMP.SPIDER # Spider temperatures (C) + CONCEPT STR FPA.TELTEMP.TRUSS # Mid truss temperatures (C + CONCEPT STR FPA.TELTEMP.EXTRA # Miscellaneous temperatures (C) + + CONCEPT STR FPA.PON.TIME # time since last power on + + STAT STR ROBUST_MEDIAN # Background estimator + STAT STR ROBUST_STDEV # Background standard deviation estimator + SUMMARY STR SAT_PIXEL_NUM + SUMMARY STR SAT_PIXEL_FRAC +END + + +# basic stats for warped images +WARPSTATS METADATA + HEADER MULTI + CONCEPT MULTI + STAT MULTI + SUMMARY MULTI + + STAT STR ROBUST_MEDIAN + STAT STR ROBUST_STDEV # Background statistics estimators + SUMMARY STR GOOD_PIXEL_NUM # Number of good pixels + SUMMARY STR GOOD_PIXEL_FRAC # Fraction of good pixels +END Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppSub.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppSub.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/ppSub.config (revision 22293) @@ -0,0 +1,31 @@ +### Recipe file for ppSub + +KERNEL.TYPE STR RINGS # Kernel type to use (POIS|ISIS|SPAM|FRIES|GUNK|RINGS) +KERNEL.SIZE S32 35 # Kernel half-size (pixels) +SPATIAL.ORDER S32 2 # Spatial polynomial order +REGION.SIZE F32 0 # Iso-kernel region size (pixels) +STAMP.SPACING F32 400 # Typical spacing between stamps (pixels) +STAMP.FOOTPRINT S32 35 # Size of stamps (pixels) +STAMP.THRESHOLD F32 0 # Flux threshold for stamps (ADU) +ITER S32 10 # Number of rejection iterations +REJ F32 1.5 # Rejection level (std dev) +MASK.BAD STR BLANK,BAD,SAT # Mask value for bad pixels +MASK.BLANK STR BLANK # Mask value to give blank pixels +BADFRAC F32 0.8 # Maximum fraction of bad pixels +@ISIS.WIDTHS F32 1 3 5 # Gaussian FWHMs for ISIS kernels +@ISIS.ORDERS S32 2 2 2 # Polynomial orders for ISIS kernels +SPAM.BINNING S32 2 # Binning in outer region for SPAM kernels +INNER S32 5 # Inner half-size for SPAM and FRIES kernels +RINGS.ORDER S32 2 # Polynomial order for RINGS kernels + +OPTIMUM BOOL FALSE # Derive optimum parameters for ISIS and GUNK kernels +OPTIMUM.MIN F32 1.0 # Minimum width for optimum FWHM search +OPTIMUM.MAX F32 11.0 # Maximum width for optimum FWHM search +OPTIMUM.STEP F32 1.0 # Step in width for optimum FWHM search +OPTIMUM.TOL F32 3.0e-3 # Maximum difference in chi^2 between iterations to settle for +OPTIMUM.ORDER S32 2 # Maximum polynomial order for optimum search + +### Modifications to use when stacking data +STACK METADATA + KERNEL.TYPE STR RINGS # Kernel type to use (POIS|ISIS|SPAM|FRIES|GUNK|RINGS) +END Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/psastro.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/psastro.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/psastro.config (revision 22293) @@ -0,0 +1,141 @@ + +PSASTRO.SAVE.REFSTARS BOOL FALSE +PSASTRO.ONLY.REFSTARS BOOL FALSE + +# perform single-chip astrometry? +PSASTRO.CHIP.MODE BOOL TRUE + +# nominal plate scale (microns / pixel) +PSASTRO.PIXEL.SCALE F32 13.5 + +# plotting options +PSASTRO.PLOT.INST.MAG.MAX F32 -10.0 +PSASTRO.PLOT.INST.MAG.MIN F32 -17.0 +PSASTRO.PLOT.REF.MAG.MIN F32 +10.0 +PSASTRO.PLOT.REF.MAG.MAX F32 +20.0 + +# extra field for ref stars: +PSASTRO.FIELD.PADDING F32 0.25 + +# pmAstromGridMatch: +PSASTRO.GRID.SEARCH BOOL TRUE + +PSASTRO.GRID.MIN.ANGLE F32 -2.0 # start angle (degrees) +PSASTRO.GRID.MAX.ANGLE F32 +2.0 +PSASTRO.GRID.DEL.ANGLE F32 0.5 + +PSASTRO.GRID.MIN.SCALE F32 0.98 +PSASTRO.GRID.MAX.SCALE F32 1.04 +PSASTRO.GRID.DEL.SCALE F32 0.02 + +PSASTRO.GRID.MIN.SIGMA F32 5.0 + +# pmAstromGridAngle +# max grid offset in FP units (microns) +# use plate-scale to make this in pixels? +PSASTRO.GRID.OFFSET F32 10000. +PSASTRO.GRID.SCALE F32 500. +PSASTRO.GRID.NSTAR.MAX S32 300 # max stars accepted for fitting + +# sources with these mask flags raised should be ignored +PSASTRO.IGNORE STR CRLIMIT,SATURATED,DEFECT,SATSTAR,BLEND,FAIL +PSASTRO.ROUGH.MODEL STR none + +# these tweak are in FP units (pixels, currently) +PSASTRO.TWEAK.SCALE F32 1 +PSASTRO.TWEAK.RANGE F32 75 +PSASTRO.TWEAK.SMOOTH F32 2 +PSASTRO.TWEAK.NSIGMA F32 3 + +# single-chip radius match in pixels +PSASTRO.MATCH.RADIUS F32 8 +PSASTRO.MATCH.RADIUS.N0 F32 0 +PSASTRO.MATCH.RADIUS.N1 F32 0 +PSASTRO.MATCH.RADIUS.N2 F32 0 +PSASTRO.MATCH.RADIUS.N3 F32 0 +PSASTRO.MATCH.RADIUS.N4 F32 0 +PSASTRO.MATCH.RADIUS.N5 F32 0 +PSASTRO.MATCH.RADIUS.N6 F32 0 +PSASTRO.MATCH.RADIUS.N7 F32 0 +PSASTRO.MATCH.FIT.NITER S32 1 + +# pmAstromMatchFit +PSASTRO.CHIP.ORDER S32 1 # fit order +PSASTRO.CHIP.NITER S32 3 # fit clipping iterations +PSASTRO.CHIP.NSIGMA F32 3 # fit clipping sigmas + +PSASTRO.MAX.ERROR F32 1.5 # max allow error for valid solution (UNITS?) +PSASTRO.MIN.NSTAR S32 3 # min fitted stars in solution +PSASTRO.MAX.NSTAR S32 300 # max fitted stars in solution + +PSASTRO.MAX.NRAW S32 0 # max stars accepted for fitting (0 for all) +PSASTRO.MAX.NREF S32 0 # max stars accepted for fitting (0 for all) + +PSASTRO.MIN.INST.MAG.RAW F32 0.0 # min instrumental magnitude for stars accepted for fitting +PSASTRO.MAX.INST.MAG.RAW F32 0.0 # max instrumental magnitude for stars accepted for fitting + +PSASTRO.MATCH.LUMFUNC BOOL FALSE + +# option may be MAX, MIN, or VALUE. for VALUE, look up +# plate scale for each chip in the concepts +PSASTRO.COMMON.SCALE.OPTION STR MAX + +# mosaic-mode radius match in pixels? +PSASTRO.MOSAIC.RADIUS.N0 F32 8 +PSASTRO.MOSAIC.RADIUS.N1 F32 6 +PSASTRO.MOSAIC.RADIUS.N2 F32 4 +PSASTRO.MOSAIC.RADIUS.N3 F32 0.0 + +# Mosaic Astrometry options +PSASTRO.MOSAIC.MODE BOOL FALSE +PSASTRO.MOSAIC.ORDER S32 3 # fit order + +PSASTRO.MOSAIC.CHIP.ORDER S32 1 # fit order +PSASTRO.MOSAIC.CHIP.ORDER.N0 S32 -1 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N1 S32 -1 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N2 S32 -1 # fit order (-1 means use default) +PSASTRO.MOSAIC.CHIP.ORDER.N3 S32 -1 # fit order (-1 means use default) + +PSASTRO.MOSAIC.CHIP.NITER S32 3 # fit clipping iterations +PSASTRO.MOSAIC.CHIP.NSIGMA F32 3 # fit clipping sigmas + +PSASTRO.MOSAIC.MAX.ERROR.N0 F32 1.5 # max allow error for valid solution (UNITS?) +PSASTRO.MOSAIC.MAX.ERROR.N1 F32 1.5 # max allow error for valid solution (UNITS?) +PSASTRO.MOSAIC.MAX.ERROR.N2 F32 1.5 # max allow error for valid solution (UNITS?) +PSASTRO.MOSAIC.MAX.ERROR.N3 F32 1.5 # max allow error for valid solution (UNITS?) +PSASTRO.MOSAIC.MIN.NSTAR S32 6 # min fitted stars in solution + +PSASTRO.MOSAIC.GRADIENT.NX S32 2 # number of x-dir cells per chip +PSASTRO.MOSAIC.GRADIENT.NY S32 2 # number of y-dir cells per chip + +# 2MASS default configuration (use 2MASS_J as ref magnitude) +DVO.CATDIR STR /data/alala.0/ipp/ippRefs/catdir.2mass +DVO.GETSTAR STR getstar +DVO.GETSTAR.OUTFORMAT STR PS1_DEV_0 +DVO.GETSTAR.PHOTCODE STR 2MASS_J +DVO.GETSTAR.MAG.MAX F32 22.0 + +PSASTRO.2MASS METADATA + DVO.CATDIR STR /data/alala.0/ipp/ippRefs/catdir.2mass + DVO.GETSTAR.PHOTCODE STR 2MASS_J + DVO.GETSTAR.MAG.MAX F32 17.0 +END + +PSASTRO.SYNTH METADATA + DVO.CATDIR STR /data/alala.0/ipp/ippRefs/catdir.synth.grizy + DVO.GETSTAR.PHOTCODE STR r +END + +LOAD.REF.ASTROM BOOL FALSE +SAVE.REF.ASTROM BOOL FALSE + +PSASTRO.MODEL.REF.CHIP STR NONE + +PSASTRO.FIX.CHIPS BOOL FALSE +PSASTRO.USE.MODEL BOOL FALSE + +PSASTRO.PIXEL.TOLERANCE F32 0.1 +PSASTRO.ANGLE.TOLERANCE F32 0.1 + +PSASTRO.FINE METADATA +END Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/psphot.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/psphot.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/psphot.config (revision 22293) @@ -0,0 +1,207 @@ + +# these options turn on different inputs and/or outputs +SAVE.OUTPUT BOOL TRUE +SAVE.BACKMDL BOOL FALSE +SAVE.BACKMDL.STDEV BOOL FALSE +SAVE.BACKGND BOOL FALSE +SAVE.BACKSUB BOOL FALSE +SAVE.RESID BOOL FALSE +SAVE.PSF BOOL FALSE +LOAD.PSF BOOL FALSE +SAVE.PLOTS BOOL FALSE + +# the zero point is used to set a basic scale for DVO +# XXX it may not currently be read : double check this (EAM) +ZERO_POINT F32 25.000 # zero point used by DVO +ZERO_PT F32 25.000 # zero point used by DVO + +MASKVAL STR SAT,BAD,BLANK # Mask these types of pixels + +OUTPUT.FORMAT STR SMPDATA + +# these parameter govern how the background is measured +BACKGROUND.XBIN S32 128 # size of background superpixels +BACKGROUND.YBIN S32 128 # size of background superpixels +IMSTATS_NPIX S32 10000 # number of pixels to use for sky estimate boxes: + +SKY_BIAS F32 0.0 # offset applied to measured sky (FOR TESTING) +SKY_FIT_ORDER S32 0 +SKY_FIT_LINEAR BOOL FALSE +SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background +SKY_CLIP_SIGMA F32 2.0 # statistic used to measure background +SKY_SIG F32 1.0 # optional sky error for + +# allowed values for SKY_STAT: +# SAMPLE_MEAN, SAMPLE_MEDIAN, CLIPPED_MEAN, ROBUST_MEAN, ROBUST_QUARTILE, FITTED_MEAN + +# masking parameters (XXX EAM : rework this to use psRegion like ANALYSIS_REGION) +XMIN F32 0 # minimum valid x-coord +XMAX F32 0 # maximum valid x-coord +YMIN F32 0 # minimum valid y-coord +YMAX F32 0 # maximum valid y-coord + +# peak finding +PEAKS_SMOOTH_SIGMA F32 1.0 # smoothing kernel sigma in pixels +PEAKS_SMOOTH_NSIGMA F32 2.0 # smoothing kernel width in sigmas +PEAKS_NSIGMA_LIMIT F32 25.0 # peak significance threshold +PEAKS_NSIGMA_LIMIT_2 F32 5.0 # peak significance threshold +PEAKS_NMAX S32 0 # on first pass, only keep NMAX peaks (0 == all) + +# parameters to control the selection of the peak in the Sx,Sy plane +MOMENTS_SCALE F32 0.25 +MOMENTS_SN_MIN F32 100.0 # min S/N to measure moments +MOMENTS_SX_MAX F32 5.0 +MOMENTS_SY_MAX F32 5.0 +MOMENTS_AR_MAX F32 1.5 # maximum axial ratio: 1 / AR < (sx / sy) < AR + +# basic object statistics +SKY_INNER_RADIUS F32 15 # square annulus for local sky measurement +SKY_OUTER_RADIUS F32 25 # square annulus for local sky measurement +PSF_MOMENTS_RADIUS F32 3 # calculate initial source moments with this radius +PSF_SN_LIM F32 50 # minimum S/N for stars used for PSF model +PSF_MAX_NSTARS S32 200 # limit number of stars used for PSF model +PSF_CLUMP_NSIGMA F32 1.5 # region of Sx,Sy plane to use for selecting PSF stars +PSF_MIN_DS F32 0.01 +PSF_PARAM_WEIGHTS BOOL FALSE + +# PSF model parameters : choose the PSF model +# list as many PSF_MODEL options as desired +# PSF_MODEL MULTI +PSF_MODEL STR PS_MODEL_GAUSS +# PSF_MODEL STR PS_MODEL_PGAUSS +# PSF_MODEL STR PS_MODEL_QGAUSS +# PSF_MODEL STR PS_MODEL_TGAUSS # not well tested, not very successful + +# PSF.TREND.MASK must be a 2D polynomial +# the specified values are ignored but define the active components of the polynomial +PSF.TREND.MASK METADATA + NORDER_X S32 0 # number of x orders + NORDER_Y S32 0 # number of y orders + VAL_X00_Y00 F64 1 # polynomial coefficient + NELEMENTS S32 1 # number of unmasked components +END # folder for 2D polynomial + +PSF.TREND.MODE STR POLY_ORD +PSF.TREND.NX S32 0 +PSF.TREND.NY S32 0 + +PSF_FIT_RADIUS F32 15.0 # fitting radius for test PSF model +PSF_REF_RADIUS F32 25.0 # aperture magnitudes are scaled via + # curve-of-growth to this radius +# PSF-like source model parameters +PSF_FIT_NSIGMA F32 1.0 # significance for pixel included in fit +PSF_FIT_PADDING F32 2.0 # extra annulus to use for fit +PSF_SHAPE_NSIGMA F32 3.0 # max significance for shape variation +PSF_MIN_SN F32 2.0 # reject objects below this significance +PSF_MAX_CHI F32 50.0 # reject objects worse that this +FULL_FIT_SN_LIM F32 50.0 + +# the RESIDUALS are pixelized psf residual tables +PSF.RESIDUALS BOOL TRUE # generate the residuals? +PSF.RESIDUALS.XBIN S32 1 # Nx(residual) = Nx(input)*XBIN +PSF.RESIDUALS.YBIN S32 1 # Ny(residual) = Ny(input)*YBIN +PSF.RESIDUALS.NSIGMA F32 3.0 # clip input stack of NSIGMA outliers +PSF.RESIDUALS.INTERPOLATION STR BILINEAR # interpolation to use when reconstructing residual +PSF.RESIDUALS.STATISTIC STR ROBUST_MEDIAN # statistic to use for generating the residual +PSF.RESIDUALS.SPATIAL_ORDER S32 0 # fit spatial variations of the residuals at this order (0,1) +PSF.RESIDUALS.PIX.SN F32 0.0 # keep this pixel if residual is more significant than this + +# EXTended source model parameters +EXT_MODEL STR PS_MODEL_PGAUSS +EXT_MIN_SN F32 50.0 # fit galaxies above this S/N limit +EXT_FIT_NSIGMA F32 1 # significance for pixel included in fit +EXT_FIT_PADDING F32 5 # extra annulus to use for fit +EXT_MOMENTS_RADIUS F32 9 + +# Extended source fit parameters +EXTENDED_SOURCE_FITS BOOL FALSE +EXTENDED_SOURCE_SN_LIM F32 20.0 +EXTENDED_SOURCE_PSF_CONVOLVED BOOL FALSE + +FITMODE STR BLEND +DEBLEND_PEAK_FRACTION F32 0.1 +DEBLEND_SKY_NSIGMA F32 10.0 + +# APTREND STR NONE, CONSTANT, SKYBIAS, SKYSAT, XY_LIN, SKY_XY_LIN, SKYSAT_XY_LIN, ALL +APTREND STR CONSTANT +AP_MIN_SN F32 25.0 +APTREND.NSTAR.MIN S32 5 +APTREND.ORDER.MAX S32 5 + +# test options +# BREAK_POINT may be one of (PEAKS, MOMENTS, PSFMODEL, ENSEMBLE) +BREAK_POINT STR NONE +# PEAKS_OUTPUT_FILE STR peaks.dat +# MOMENTS_OUTPUT_FILE STR moments.dat +# ANALYSIS_REGION STR [1000:1600,2800:3400] + +# optional parameter to limit the actual analysis to a fraction of the image +# do not uncomment this in the master psphot.config file +# ANALYSIS_REGION STR [1000:1600,2800:3400] + +IGNORE_GROWTH BOOL FALSE +CONSTANT_PHOTOMETRIC_WEIGHTS BOOL TRUE # Should the photometric code [currently only ensemblePSF] refuse to weight each pixel by it's significance? +INTERPOLATE_AP BOOL TRUE + +POISSON.ERRORS.PHOT.LMM BOOL TRUE +POISSON.ERRORS.PHOT.LIN BOOL FALSE +POISSON.ERRORS.PARAMS BOOL TRUE + +PCM_BOX_SIZE S32 2 + +OUTPUT.FORMAT STR SMPDATA +NOISE.FACTOR F32 5.0 +NOISE.SIZE F32 2.0 + +USE_FOOTPRINTS BOOL F # use new pmFootprint peak packaging +FOOTPRINT_NPIXMIN S32 5 # Minimum size of a pmFootprint +FOOTPRINT_NSIGMA_LIMIT F32 20 # threshold for bright pmFootprint detection +FOOTPRINT_NSIGMA_LIMIT_2 F32 4 # threshold for faint pmFootprint detection +FOOTPRINT_GROW_RADIUS S32 3 # How much to grow bright footprints +FOOTPRINT_GROW_RADIUS_2 S32 5 # How much to grow faint footprints +FOOTPRINT_CULL_NSIGMA_DELTA F32 4 # Cull peaks that aren't nsigma above coll to neighbour +FOOTPRINT_CULL_NSIGMA_MIN F32 1 # Minimum height of colls in units of skyStdev + +# alternate PSPHOT-type recipes +PSPHOT.FIXED.PSF METADATA +END + +# alternate PSPHOT-type recipes +PSPHOT.SUMMIT METADATA +END + +# alternate PSPHOT-type recipes +PSPHOT.SEEING METADATA +END + +TEST_FIT BOOL FALSE +TEST_FIT_MODE STR DEFAULT +TEST_FIT_MODEL STR DEFAULT +TEST_FIT_INNER_RADIUS F32 NAN +TEST_FIT_OUTER_RADIUS F32 NAN +TEST_FIT_RADIUS F32 NAN +TEST_MOMENTS_RADIUS F32 NAN +TEST_FIT_X F32 NAN +TEST_FIT_Y F32 NAN + +TEST_FIT_PAR0 F32 NAN +TEST_FIT_PAR1 F32 NAN +TEST_FIT_PAR2 F32 NAN +TEST_FIT_PAR3 F32 NAN +TEST_FIT_PAR4 F32 NAN +TEST_FIT_PAR5 F32 NAN +TEST_FIT_PAR6 F32 NAN +TEST_FIT_PAR7 F32 NAN +TEST_FIT_PAR8 F32 NAN +TEST_FIT_PAR F32 NAN + +DIAGNOSTIC.PLOTS METADATA + IMAGE.BACKGROUND.CELL.HISTOGRAM BOOL FALSE + IMAGE.BACKGROUND.CELL.HISTOGRAM.X S32 35 + IMAGE.BACKGROUND.CELL.HISTOGRAM.Y S32 -1 +END + +PSF.FLUXSCALE.NX S32 5 # number cells to measure flux scale variations +PSF.FLUXSCALE.NY S32 5 # number cells to measure flux scale variations + +PSPHOT.CRNSIGMA.LIMIT F32 3.0 # sources with crNsigma greater that this get tagged as likely cosmic rays Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/pswarp.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/pswarp.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/pswarp.config (revision 22293) @@ -0,0 +1,13 @@ +### Recipe for pswarp + +GRID.NX S32 128 # Iso-astrom grid size in x (pixels) +GRID.NY S32 128 # Iso-astrom grid size in y (pixels) +INTERPOLATION.MODE STR LANCZOS3 # Interpolation mode to use +MASK.IN STR BAD,SAT # Mask for input data +MASK.POOR STR SUSPECT # Mask for "poor" warped data +MASK.BAD STR BLANK # Mask for bad warped data +POOR.FRAC F32 0.01 # Max fraction of bad flux for a "poor" warped pixel +ASTROM.SOURCE STR PSASTRO.OUTPUT.MEF # Source file rule for astrometry, or NULL +ASTROM.ACCEPT BOOL FALSE # Accept astrometric solution unconditionally? +ASTROM.DEPTH STR MEF # Source file rule for astrometry, or NULL +ACCEPT.FRAC F32 0.1 # Minimum fraction of good pixels to accept result Index: /tags/pap_tags/pap_root_080320/ippconfig/recipes/rejections.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/recipes/rejections.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/recipes/rejections.config (revision 22293) @@ -0,0 +1,182 @@ + +BIAS METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 2.0 + IMFILE.STDEV F32 10.0 + IMFILE.MEANSTDEV F32 0.5 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.5 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 1.0 + EXP.STDEV F32 10.0 + EXP.MEANSTDEV F32 0.5 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 3.0 + ENSEMBLE.STDEV F32 3.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +DARK METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 2.0 + IMFILE.STDEV F32 10.0 + IMFILE.MEANSTDEV F32 0.5 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 1.0 + EXP.STDEV F32 10.0 + EXP.MEANSTDEV F32 0.5 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 3.0 + ENSEMBLE.STDEV F32 3.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +SHUTTER METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 20.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +FLAT METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 3.0 +END + +FRINGE METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +DARKMASK METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +FLATMASK METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + +# FILTER is an additional qualifier, and may be "*" (or absent!), in which case it matches everything +# EXPECTED is the expected mean value +# IMFILE.MEAN is the maximum permitted mean value for an imfile, relative to the standard deviation +# IMFILE.STDEV is the maximum permitted standard deviation for an imfile +# EXP.MEAN is the maximum permitted mean value for an exposure, relative to the standard deviation +# EXP.STDEV is the maximum permitted standard deviation for an exposure +# EXP.MEANSTDEV is the maximum permitted mean standard deviation for an exposure relative to the mean +# ENSEMBLE.MEAN is the maximum permitted mean for an ensemble of exposures +# ENSEMBLE.STDEV is the maximum permitted standard deviation for an ensemble of exposures +# ENSEMBLE.MEANSTDEV is the maximum permitted mean standard deviation for an ensemble of exposures +# IMFILE.SNR is the minimum permitted signal-to-noise for an imfile +# EXP.SNR is the minimum permitted signal-to-noise for an exposure +# These values (all except FILTER) may be zero, in which case no clipping is applied. Index: /tags/pap_tags/pap_root_080320/ippconfig/simmosaic/camera.config =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/simmosaic/camera.config (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/simmosaic/camera.config (revision 22293) @@ -0,0 +1,376 @@ +# Camera configuration file for simulated mosaic + +# File formats that we know about +FORMATS METADATA + TOGETHER STR simmosaic/format_together.config + SPLIT STR simmosaic/format_split.config +END + +# Description of camera --- all the chips and the cells that comprise them +FPA METADATA + Chip00 STR Cell00 Cell01 Cell10 Cell11 + Chip01 STR Cell00 Cell01 Cell10 Cell11 + Chip10 STR Cell00 Cell01 Cell10 Cell11 + Chip11 STR Cell00 Cell01 Cell10 Cell11 +END + +# move this elsewhere? we need a lookup table to go from filter ID to abstract name +FILTER.ID METADATA + NONE STR NONE + B STR B + V STR V + R STR R + I STR I + g STR g + r STR r + i STR i + z STR z +END + +# Table of strings to use for the class identifier (see ippdb) according to the file level. +CLASSID METADATA + FPA STR fpa + CHIP STR {CHIP.NAME} + CELL STR {CHIP.NAME}.{CELL.NAME} + fpa STR fpa + chip STR {CHIP.NAME} + cell STR {CHIP.NAME}.{CELL.NAME} +END + +DVO.CAMERADIR STR simmosaic # Camera directory for DVO + +# convert supplied FPA.OBSTYPE values to abstract exptype names +OBSTYPE.TABLE METADATA + bias STR BIAS + zero STR BIAS + dark STR DARK + flat STR SKYFLAT + skyflat STR SKYFLAT + domeflat STR DOMEFLAT + object STR OBJECT + science STR OBJECT +END + +# Recipe options +RECIPES METADATA + PPIMAGE STR simmosaic/ppImage.config # Default: all (normal) options on + PSPHOT STR simmosaic/psphot.config # psphot details + PSASTRO STR simmosaic/psastro.config # psastro details + REJECTIONS STR simmosaic/rejections.config # Rejection limits +END + +# Reduction classes +REDUCTION METADATA + # Detrend processing + DETREND METADATA + BIAS_PROCESS STR PPIMAGE_O + BIAS_RESID STR PPIMAGE_B + BIAS_VERIFY STR PPIMAGE_OB + BIAS_STACK STR PPMERGE_BIAS + DARK_PROCESS STR PPIMAGE_OB + DARK_RESID STR PPIMAGE_D + DARK_VERIFY STR PPIMAGE_OBD + DARK_STACK STR PPMERGE_DARK + SHUTTER_PROCESS STR PPIMAGE_OBD + SHUTTER_RESID STR PPIMAGE_S + SHUTTER_VERIFY STR PPIMAGE_OBDS + SHUTTER_STACK STR PPMERGE_SHUTTER + FLAT_PROCESS STR PPIMAGE_OBDS + FLAT_RESID STR PPIMAGE_F + FLAT_VERIFY STR PPIMAGE_OBDSF + FLAT_STACK STR PPMERGE_FLAT + FRINGE_PROCESS STR PPIMAGE_OBDSF + FRINGE_RESID STR PPIMAGE_R + FRINGE_VERIFY STR PPIMAGE_OBDSFR + FRINGE_STACK STR PPMERGE_FRINGE + + # Generation of pixel masks from darks and flats + DARKMASK_PROCESS STR PPIMAGE_OBD + DARKMASK_RESID STR PPIMAGE_N + DARKMASK_VERIFY STR PPIMAGE_OBD + DARKMASK_STACK STR PPMERGE_DARKMASK + FLATMASK_PROCESS STR PPIMAGE_OBDSF + FLATMASK_RESID STR PPIMAGE_N + FLATMASK_VERIFY STR PPIMAGE_OBDSF + FLATMASK_STACK STR PPMERGE_FLATMASK + JPEG_BIN1_IMAGE_DARKMASK STR PPIMAGE_J1_IMAGE_M + JPEG_BIN2_IMAGE_DARKMASK STR PPIMAGE_J2_IMAGE_M + JPEG_BIN1_IMAGE_FLATMASK STR PPIMAGE_J1_IMAGE_M + JPEG_BIN2_IMAGE_FLATMASK STR PPIMAGE_J2_IMAGE_M + JPEG_BIN1_RESID_DARKMASK STR PPIMAGE_J1_RESID_M + JPEG_BIN2_RESID_DARKMASK STR PPIMAGE_J2_RESID_M + JPEG_BIN1_RESID_FLATMASK STR PPIMAGE_J1_RESID_M + JPEG_BIN2_RESID_FLATMASK STR PPIMAGE_J2_RESID_M + + JPEG_BIN1_IMAGE_BIAS STR PPIMAGE_J1_IMAGE_B + JPEG_BIN1_IMAGE_DARK STR PPIMAGE_J1_IMAGE_B + JPEG_BIN1_IMAGE_SHUTTER STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_FLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_DOMEFLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_SKYFLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_FRINGE STR PPIMAGE_J1_IMAGE_R + JPEG_BIN2_IMAGE_BIAS STR PPIMAGE_J2_IMAGE_B + JPEG_BIN2_IMAGE_DARK STR PPIMAGE_J2_IMAGE_B + JPEG_BIN2_IMAGE_SHUTTER STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_FLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_DOMEFLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_SKYFLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_FRINGE STR PPIMAGE_J2_IMAGE_R + + JPEG_BIN1_RESID_BIAS STR PPIMAGE_J1_RESID_B + JPEG_BIN1_RESID_DARK STR PPIMAGE_J1_RESID_B + JPEG_BIN1_RESID_SHUTTER STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_FLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_DOMEFLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_SKYFLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_FRINGE STR PPIMAGE_J1_RESID_R + JPEG_BIN2_RESID_BIAS STR PPIMAGE_J2_RESID_B + JPEG_BIN2_RESID_DARK STR PPIMAGE_J2_RESID_B + JPEG_BIN2_RESID_SHUTTER STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_FLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_DOMEFLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_SKYFLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_FRINGE STR PPIMAGE_J2_RESID_R + END + # Processing raw data + DEFAULT METADATA + CHIP STR PPIMAGE_OBDSFRA + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + END +END + +FITS METADATA +# BITPIX is the bits per pixel for writing the output data +# COMP = NONE|RICE|GZIP|HCOMPRESS|PLIO is the compression algorithm +# TILE.[XYZ] are the tile sizes. 0 means entire the dimension, so (0,1,1) forms tiles from rows +# NOISE [0..16] is the number of "noise bits" to preserve when quantising floating point data; 16 for no loss +# HSCALE is the scale factor for lossy compression with HCOMPRESS; 0 or 1 for none; 2*RMS --> 10x compression +# HSMOOTH is the smoothing to apply to HCOMPRESSed data when decompressing; 0 for none + +# BITPIX(S32) is the bits per pixel for writing the output data +# COMPRESSION(STR) = NONE|RICE|GZIP|HCOMPRESS|PLIO is the compression algorithm +# TILE.[XYZ](S32) are the tile sizes. 0 means entire the dimension, so (0,1,1) forms tiles from rows +# NOISE(S32) [0..16] is the number of "noise bits" to preserve when quantising floating point data +# HSCALE(S32) is the scale factor for lossy compression with HCOMPRESS; 0 or 1 for none; 2*RMS --> 10x compression +# HSMOOTH(S32) is the smoothing to apply to HCOMPRESSed data when decompressing; 0 for none +# SCALING(STR) = NONE|RANGE|STDEV_POSITIVE|STDEV_NEGATIVE|STDEV_BOTH|MANUAL is the scaling scheme +# BSCALE(F32) is the manual scaling to apply (when SCALING = MANUAL) +# BZERO(F32) is the manual zero-point to apply (when SCALING = MANUAL) +# STDEV.BITS(S32) is the number of bits to map to a standard deviation (when SCALING = STDEV_*) +# STDEV.NUM(F32) is the number of standard deviations to the edge (when SCALING = STDEV_NEGATIVE|STDEV_POSITIVE) +# FLOAT(STR) is the name of a custom floating-point type + + DET_IMAGE METADATA + BITPIX S32 -32 + END + DET_MASK METADATA + BITPIX S32 8 + END + DET_WEIGHT METADATA + BITPIX S32 -32 + END + + SKY_IMAGE METADATA + BITPIX S32 -32 + END + SKY_MASK METADATA + BITPIX S32 8 + END + SKY_WEIGHT METADATA + BITPIX S32 -32 + END + + COMPRESSED_POSITIVE METADATA + BITPIX S32 16 + SCALING STR STDEV_POSITIVE + STDEV.BITS S32 4 + STDEV.NUM F32 10 + COMPRESSION STR RICE + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + COMPRESSED_MASK METADATA + COMPRESSION STR PLIO + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + COMPRESSED_SUBTRACTION METADATA + BITPIX S32 16 + SCALING STR STDEV_BOTH + STDEV.BITS S32 4 + STDEV.NUM F32 5 + COMPRESSION STR RICE + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + +END + +FILERULES METADATA + ### Redirections + PSASTRO.INPUT STR PSASTRO.INPUT.CMF + PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.SPL + PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF + PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.SPL + + ### input file definitions + ### use @DETDB entries to get the detrend images from the database + ### replace @DETDB with @FILES if you want to require it from the + ### command line, or with an explicit name to require a specific file + TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + + ## files used by ppImage + PPIMAGE.INPUT INPUT @FILES CHIP IMAGE + PPIMAGE.MASK INPUT @DETDB CELL IMAGE + PPIMAGE.BIAS INPUT @DETDB CELL IMAGE + PPIMAGE.DARK INPUT @DETDB CELL DARK + PPIMAGE.FLAT INPUT @DETDB CELL IMAGE + PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE + PPIMAGE.SHUTTER INPUT @DETDB CELL IMAGE + + ## files used to build and apply the flat-field correction images + DVOCORR.INPUT INPUT @FILES CHIP IMAGE + DVOCORR.REFHEAD INPUT @FILES CHIP HEADER + DVOFLAT.INPUT INPUT @FILES CHIP IMAGE + DVOFLAT.CORR INPUT @DETDB CHIP IMAGE + + ## files used by psphot + PSPHOT.LOAD INPUT @FILES CHIP IMAGE + PSPHOT.INPUT INPUT @FILES CHIP IMAGE + PSPHOT.MASK INPUT @FILES CHIP MASK + PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT + PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF + PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + + ## files used by psastro + PSASTRO.INPUT.CMF INPUT @FILES CHIP CMF + + ## files used by pswarp + PSWARP.INPUT INPUT @FILES CHIP IMAGE + PSWARP.SKYCELL INPUT @FILES FPA IMAGE + PSWARP.WEIGHT INPUT @FILES CHIP WEIGHT + PSWARP.MASK INPUT @FILES CHIP MASK + PSWARP.ASTROM INPUT @FILES CHIP CMF + + PPSUB.INPUT INPUT none.fits FPA IMAGE + PPSUB.INPUT.MASK INPUT none.fits FPA MASK + PPSUB.INPUT.WEIGHT INPUT none.fits FPA WEIGHT + PPSUB.REF INPUT none.fits FPA IMAGE + PPSUB.REF.MASK INPUT none.fits FPA MASK + PPSUB.REF.WEIGHT INPUT none.fits FPA WEIGHT + + PPSTACK.INPUT INPUT none.fits FPA IMAGE + PPSTACK.INPUT.MASK INPUT none.fits FPA MASK + PPSTACK.INPUT.WEIGHT INPUT none.fits FPA WEIGHT + + PPSTAMP.INPUT INPUT @FILES CHIP IMAGE + + PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE + PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + + + ### output file definitions + TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT + PPIMAGE.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT + PPIMAGE.OUTPUT.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.mask.fits MASK NONE CHIP TRUE SPLIT + PPIMAGE.OUTPUT.WEIGHT OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT NONE CHIP TRUE SPLIT + PPIMAGE.CHIP OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE NONE CHIP TRUE SPLIT + PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.ch.mask.fits MASK NONE CHIP TRUE SPLIT + PPIMAGE.CHIP.WEIGHT OUTPUT {OUTPUT}.{CHIP.NAME}.ch.wt.fits WEIGHT NONE CHIP TRUE SPLIT + PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.fpa1.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.OUTPUT.FPA2 OUTPUT {OUTPUT}.fpa2.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + + PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE + PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE + PPIMAGE.BIN1 OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE NONE CHIP TRUE SPLIT + PPIMAGE.BIN2 OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE NONE CHIP TRUE SPLIT + + PPMERGE.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE + + DVOCORR.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE NONE CHIP TRUE NONE + DVOFLAT.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE NONE CHIP TRUE NONE + + PSPHOT.RESID OUTPUT {OUTPUT}.{CHIP.NAME}.res.fits IMAGE NONE CHIP TRUE NONE + PSPHOT.BACKGND OUTPUT {OUTPUT}.{CHIP.NAME}.bck.fits IMAGE NONE CHIP TRUE NONE + PSPHOT.BACKSUB OUTPUT {OUTPUT}.{CHIP.NAME}.sub.fits IMAGE NONE CHIP TRUE NONE + PSPHOT.BACKMDL OUTPUT {OUTPUT}.{CHIP.NAME}.mdl.fits IMAGE NONE CHIP TRUE NONE + PSPHOT.BACKMDL.STDEV OUTPUT {OUTPUT}.{CHIP.NAME}.mdd.fits IMAGE NONE CHIP TRUE NONE + + PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW NONE CHIP TRUE NONE + PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.{CHIP.NAME}.sx SX NONE CHIP TRUE NONE + PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.{CHIP.NAME}.obj OBJ NONE CHIP TRUE NONE + PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF NONE CHIP TRUE NONE + PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE + + PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.{CHIP.NAME}.psf PSF NONE CHIP TRUE NONE + + SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.{CHIP.NAME}.mnt.png KAPA NONE CHIP TRUE NONE + SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.{CHIP.NAME}.psf.png KAPA NONE CHIP TRUE NONE + SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.{CHIP.NAME}.dap.png KAPA NONE CHIP TRUE NONE + + PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE SPLIT + PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE SPLIT + PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE TOGETHER + + PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE + PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE + PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + + SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + + PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE + PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + + PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE + PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.weight.fits WEIGHT NONE FPA TRUE NONE + + PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + + PPSIM.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT + PPSIM.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE + + PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE + PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE + + LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE + LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE +END + +# FPA file defines properties of a possible input|output object +# user can set the filename (I|O), filename rules (O), or abstract source (@FILES, @DETDB) (I) +# user can set the extension name, if used +# user can set the file type (IMAGE, JPEG, RAW, SX, OBJ, CMP, CMF) : but these are not variable in most cases! +# user can set the file depth: only valid for output files +# user can set the data depth: must be >= file depth +# user can set the file format: only valid for newly created FPAs +# user can set the colormap, scaling method, scaling range (JPEG only) +# user can set the extension name for the data and header segments (CMF only) + + +EXTNAME.RULES METADATA + CMF.HEAD STR {CHIP.NAME}.hdr + CMF.DATA STR {CHIP.NAME}.psf + PSF.HEAD STR {CHIP.NAME}.hdr + PSF.TABLE STR {CHIP.NAME}.psf_model + PSF.RESID STR {CHIP.NAME}.psf_resid +END + +BLANK.HEADERS METADATA + FPA.TIME STR MJD-OBS + FPA.EXPOSURE STR EXPTIME + FPA.AIRMASS STR AIRMASS +END Index: /tags/pap_tags/pap_root_080320/ippconfig/simtest/filerules.mdc =================================================================== --- /tags/pap_tags/pap_root_080320/ippconfig/simtest/filerules.mdc (revision 22293) +++ /tags/pap_tags/pap_root_080320/ippconfig/simtest/filerules.mdc (revision 22293) @@ -0,0 +1,124 @@ +### File rules for PHU=FPA, EXT=NONE + + +PSASTRO.INPUT STR PSASTRO.INPUT.CMF +PSASTRO.OUTPUT STR PSASTRO.OUTPUT.CMF +PSASTRO.OUTPUT.MEF STR PSASTRO.OUTPUT.CMF +PSPHOT.OUTPUT STR PSPHOT.OUTPUT.CMF + +### input file definitions +TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE +PPIMAGE.INPUT INPUT none.fits FPA IMAGE +INPUT.MASK INPUT none.fits FPA MASK +INPUT.WEIGHT INPUT none.fits FPA WEIGHT +INPUT.PSF INPUT none.fits READOUT PSF +INPUT.SRC INPUT none.fits READOUT CMF +PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE +PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE +PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE +PPIMAGE.MASK INPUT @DETDB CHIP IMAGE + +## files used by psphot +PSPHOT.LOAD INPUT @FILES CHIP IMAGE +PSPHOT.INPUT INPUT none.fits CHIP IMAGE + +PSWARP.INPUT INPUT none.fits FPA IMAGE +PSWARP.WEIGHT INPUT none.fits FPA WEIGHT +PSWARP.MASK INPUT none.fits FPA MASK +PSWARP.SKYCELL INPUT none.fits FPA IMAGE +PSWARP.ASTROM INPUT none.fits FPA CMF + +PSASTRO.INPUT.CMP INPUT none.fits CHIP CMP +PSASTRO.INPUT.CMF INPUT none.fits CHIP CMF +PSPHOT.PSF.LOAD INPUT none.psf CHIP PSF +PSPHOT.INPUT.CMF INPUT none.fits CHIP CMF + +PPSUB.INPUT INPUT none.fits FPA IMAGE +PPSUB.INPUT.MASK INPUT none.fits FPA MASK +PPSUB.INPUT.WEIGHT INPUT none.fits FPA WEIGHT +PPSUB.REF INPUT none.fits FPA IMAGE +PPSUB.REF.MASK INPUT none.fits FPA MASK +PPSUB.REF.WEIGHT INPUT none.fits FPA WEIGHT + +PPSTACK.INPUT INPUT none.fits FPA IMAGE +PPSTACK.INPUT.MASK INPUT none.fits FPA MASK +PPSTACK.INPUT.WEIGHT INPUT none.fits FPA WEIGHT + +PPSTAMP.INPUT INPUT none.fits FPA IMAGE + +PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE +PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + +### output file definitions +TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT +PPIMAGE.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE +PPIMAGE.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE +PPIMAGE.CHIP OUTPUT {OUTPUT}.ch.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.ch.mask.fits MASK NONE FPA TRUE NONE +PPIMAGE.CHIP.WEIGHT OUTPUT {OUTPUT}.ch.wt.fits WEIGHT NONE FPA TRUE NONE +PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.OUTPUT.FPA2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + +PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + +PPMERGE.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + +DVOCORR.OUTPUT OUTPUT {OUTPUT}.fc.fits IMAGE NONE FPA TRUE NONE +DVOFLAT.OUTPUT OUTPUT {OUTPUT}.co.fits IMAGE NONE FPA TRUE NONE + +PSPHOT.RESID OUTPUT {OUTPUT}.res.fits IMAGE NONE FPA TRUE NONE +PSPHOT.BACKGND OUTPUT {OUTPUT}.bck.fits IMAGE NONE FPA TRUE NONE +PSPHOT.BACKSUB OUTPUT {OUTPUT}.sub.fits IMAGE NONE FPA TRUE NONE +PSPHOT.BACKMDL OUTPUT {OUTPUT}.mdl.fits IMAGE NONE FPA TRUE NONE + +PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT} RAW NONE FPA TRUE NONE +PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.sx SX NONE FPA TRUE NONE +PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.obj OBJ NONE FPA TRUE NONE +PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.cmp CMP NONE FPA TRUE NONE +PSPHOT.OUTPUT.CMF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE +PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE FPA TRUE NONE + +SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.mnt.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.psf.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.dap.png KAPA NONE FPA TRUE NONE + +PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.smp CMP NONE FPA TRUE NONE +PSASTRO.OUTPUT.CMF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE NONE + +PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE +PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE +PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE +PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + +SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE +SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + +PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE +PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + +PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE +PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.weight.fits WEIGHT NONE FPA TRUE NONE + +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.CHIP OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + +PPSIM.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE SIMTEST +PPSIM.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE + +PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE + +LOG.IMFILE OUTPUT {OUTPUT}.imfile.log TEXT NONE FPA TRUE NONE +LOG.EXP OUTPUT {OUTPUT}.exp.log TEXT NONE FPA TRUE NONE + +TRACE.IMFILE OUTPUT {OUTPUT}.imfile.trace TEXT NONE FPA TRUE NONE +TRACE.EXP OUTPUT {OUTPUT}.exp.trace TEXT NONE FPA TRUE NONE Index: /tags/pap_tags/pap_root_080320/ppImage/src/ppImageDetrendReadout.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppImage/src/ppImageDetrendReadout.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppImage/src/ppImageDetrendReadout.c (revision 22293) @@ -0,0 +1,113 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +bool ppImageDetrendReadout(pmConfig *config, ppImageOptions *options, pmFPAview *view) +{ + // construct a view for the detrend images (which have only one readout) + pmFPAview *detview = pmFPAviewAlloc(0); + *detview = *view; + detview->readout = 0; + + // find the currently selected readout + pmReadout *input = pmFPAfileThisReadout(config->files, view, "PPIMAGE.INPUT"); + + // Masking on the basis of pixel value needs to be done before anything else, so the values are pristine. + if (options->doMaskBuild) { + pmReadoutGenerateMask(input, options->satMask, options->badMask); + } + // apply the externally supplied mask to the input->mask pixels + if (options->doMask) { + pmReadout *mask = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.MASK"); + pmMaskBadPixels(input, mask, options->maskValue); + } + +# if 0 + // Non-linearity correction + if (options->doNonLin) { + ppImageDetrendNonLinear(detrend->input, input, options); + } +# endif + + // set up the dark and bias + pmCell *dark = NULL; // Multi-dark + pmReadout *oldDark = NULL; // Old-fashioned dark + pmReadout *bias = NULL; + if (options->doBias) { + bias = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.BIAS"); + } + if (options->doDark) { + bool mdok; // Status of MD lookup + psMetadata *recipe = psMetadataLookupPtr (&mdok, config->recipes, RECIPE_NAME); + assert(mdok && recipe); + if (psMetadataLookupBool(&mdok, recipe, "OLDDARK")) { + oldDark = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.DARK"); + } else { + dark = pmFPAfileThisCell(config->files, detview, "PPIMAGE.DARK"); + } + } + + // Bias, dark and overscan subtraction are all merged. + if (options->doBias || options->doOverscan) { + if (!pmBiasSubtract(input, options->overscan, bias, oldDark, view)) { + psError(PS_ERR_UNKNOWN, false, "Unable to subtract bias."); + return false; + } + } + + // Weight on the basis of pixel value needs to be done after the overscan has been subtracted + if (options->doWeightBuild) { + // create the target mask and weight images + pmReadoutGenerateWeight(input, true); + } + + if (options->doDark && dark) { + if (!pmDarkApply(input, dark, options->maskValue)) { + psError(PS_ERR_UNKNOWN, false, "Unable to subtract dark."); + return false; + } + } + + // Shutter correction + if (options->doShutter) { + pmReadout *shutter = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.SHUTTER"); + if (!pmShutterCorrectionApply(input, shutter)) { + return false; + } + } + + // Flat-field correction (no options used?) + if (options->doFlat) { + pmReadout *flat = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.FLAT"); + if (!pmFlatField(input, flat, options->flatMask)) { + return false; + } + } + + // Normalisation by a (known) constant + bool mdok; // Status of MD lookup + float norm = psMetadataLookupF32(&mdok, config->arguments, "NORMALISATION"); + if (mdok && isfinite(norm) && norm != 1.0) { + pmHDU *hdu = pmHDUFromReadout(input); // HDU of interest + psString comment = NULL; // Comment to add + psStringAppend(&comment, "Normalization: %f", norm); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + + psBinaryOp(input->image, input->image, "*", psScalarAlloc(norm, PS_TYPE_F32)); + } + + if (options->doFringe) { + pmCell *fringe = pmFPAfileThisCell(config->files, detview, "PPIMAGE.FRINGE"); + if (!ppImageDetrendFringeMeasure(input, fringe, false, options)) { + return false; + } + } + + ppImageMemoryDump("detrend"); + + psFree(detview); + return true; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/.cvsignore =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/.cvsignore (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/.cvsignore (revision 22293) @@ -0,0 +1,12 @@ +Makefile +Makefile.in +aclocal.m4 +autom4te.cache +compile +config.log +config.status +configure +depcomp +install-sh +missing +test Index: /tags/pap_tags/pap_root_080320/ppMerge/Makefile.am =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/Makefile.am (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/Makefile.am (revision 22293) @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = *~ core core.* Index: /tags/pap_tags/pap_root_080320/ppMerge/autogen.sh =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/autogen.sh (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/autogen.sh (revision 22293) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ppMerge +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +#($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $LIBTOOlIZE installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" +# DIE=1 +#} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOHEADER installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +#$LIBTOOLIZE --copy --force || echo "$LIBTOOlIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /tags/pap_tags/pap_root_080320/ppMerge/configure.ac =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/configure.ac (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/configure.ac (revision 22293) @@ -0,0 +1,30 @@ +AC_PREREQ(2.61) + +AC_INIT([ppMerge], [1.1.0], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([src]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDCFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC_C99 +AC_PROG_INSTALL +dnl AC_PROG_LIBTOOL +AC_SYS_LARGEFILE + +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0]) +PKG_CHECK_MODULES([PSMODULE], [psmodules >= 1.0.0]) +PKG_CHECK_MODULES([PPSTATS], [ppStats >= 1.0.0]) + +IPP_STDOPTS +CFLAGS="${CFLAGS=} -Wall -Werror" + +AC_CONFIG_FILES([ + Makefile + src/Makefile +]) +AC_OUTPUT Index: /tags/pap_tags/pap_root_080320/ppMerge/src/.cvsignore =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/.cvsignore (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/.cvsignore (revision 22293) @@ -0,0 +1,7 @@ +.deps +Makefile +Makefile.in +ppMerge +config.h +config.h.in +stamp-h1 Index: /tags/pap_tags/pap_root_080320/ppMerge/src/Makefile.am =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/Makefile.am (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/Makefile.am (revision 22293) @@ -0,0 +1,40 @@ +bin_PROGRAMS = ppMerge + +ppMerge_CFLAGS = $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) +ppMerge_LDFLAGS = $(PPSTATS_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS) + +ppMerge_SOURCES = \ + ppMerge.c \ + ppMergeCheckInputs.c \ + ppMergeCombine.c \ + ppMergeConfig.c \ + ppMergeData.c \ + ppMergeMask.c \ + ppMergeMaskSuspect.c \ + ppMergeMaskBad.c \ + ppMergeMaskWrite.c \ + ppMergeMaskGrow.c \ + ppMergeMaskAverageConcepts.c \ + ppMergeOptions.c \ + ppMergeScaleZero.c \ + ppMergeVersion.c + + +noinst_HEADERS = \ + ppMerge.h \ + ppMergeCheckInputs.h \ + ppMergeCombine.h \ + ppMergeConfig.h \ + ppMergeData.h \ + ppMergeMask.h \ + ppMergeOptions.h \ + ppMergeScaleZero.h \ + ppMergeVersion.h + +CLEANFILES = *~ + +clean-local: + -rm -f TAGS +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMerge.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMerge.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMerge.c (revision 22293) @@ -0,0 +1,99 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeConfig.h" +#include "ppMergeData.h" +#include "ppMergeOptions.h" +#include "ppMergeCheckInputs.h" +#include "ppMergeCombine.h" +#include "ppMergeScaleZero.h" +#include "ppMergeMask.h" + +//#include "ppMem.h" + +// Yet to do: +// +// 1. Mask pixels with less than the minimum number of electrons +// 2. Sampling factor for background measurement +// 3. On/off pairs + +int main(int argc, char **argv) +{ + psLibInit(NULL); + psMemSetThreadSafety(false); // Turn off thread safety, for more + psTimerStart(TIMERNAME); + + // Parse the configuration and arguments + // Open the input image(s) + // Determine camera, format from header if not already defined + // Construct camera in preparation for reading + pmConfig *config = ppMergeConfig(argc, argv); + if (!config) { + psErrorStackPrint(stderr, "Unable to get configuration."); + exit(EXIT_FAILURE); + } + + // Set various tasks (define optional operations) + ppMergeOptions *options = ppMergeOptionsParse(config); + if (!options) { + psErrorStackPrint(stderr, "Unable to parse options."); + exit(EXIT_FAILURE); + } + + // Check the inputs + ppMergeData *data = ppMergeCheckInputs(options, config); + if (!data) { + psErrorStackPrint(stderr, "Not enough valid input files."); + exit(EXIT_FAILURE); + } + + if (options->mask) { + // Generate a mask + ppMergeMask(data, options, config); + } else { + + psImage *scale = NULL; // The scalings + psImage *zero = NULL; // The zeros + psArray *shutters = NULL; // The shutter correction data + + // Measure the background in each image + ppMergeScaleZero(&scale, &zero, &shutters, data, options, config); + + // Do the combination and write + ppMergeCombine(scale, zero, shutters, data, options, config); + + psFree(scale); + psFree(zero); + psFree(shutters); + } + + // Output the statistics + if (data->statsFile && data->stats) { + psString statsOut = psMetadataConfigFormat(data->stats); // String to write out + fprintf(data->statsFile, "%s", statsOut); + psFree(statsOut); + } + +#if 0 + pmFPAPrint(stdout, data->out, true, true); +#endif + + // Cleaning up + psFree(data); + psFree(options); + psFree(config); + + pmConceptsDone(); + pmConfigDone(); + psLibFinalize(); + +// ppMemCheck(); + + return EXIT_SUCCESS; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMerge.h =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMerge.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMerge.h (revision 22293) @@ -0,0 +1,7 @@ +#ifndef PP_MERGE_H +#define PP_MERGE_H + +#define TIMERNAME "ppMerge" +#define PPMERGE_RECIPE "PPMERGE" + +#endif Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeCheckInputs.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeCheckInputs.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeCheckInputs.c (revision 22293) @@ -0,0 +1,177 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeCheckInputs.h" +#include "ppMergeData.h" + +// Check input files to make sure everything's consistent +ppMergeData *ppMergeCheckInputs(ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + ppMergeData *data = ppMergeDataAlloc(); // The data, to return + + // Output file + psString outName = psMetadataLookupStr(NULL, config->arguments, "OUTPUT"); // The output file name + assert(outName); // It should be there! + outName = pmConfigConvertFilename(outName, config, true); + data->outFile = psFitsOpen(outName, "w"); // Output FITS file + if (!data->outFile) { + // There's no point in continuing if we can't open the output + psErrorStackPrint(stderr, "Can't open output image: %s\n", outName); + psFree(outName); + exit(EXIT_FAILURE); + } + psFree(outName); + + // Statistics file + psString statsName = psMetadataLookupStr(NULL, config->arguments, "-stats"); // Name for statistics file + if (statsName && strlen(statsName) > 0) { + psString resolved = pmConfigConvertFilename(statsName, config, true); // Resolved filename + data->statsFile = fopen(resolved, "w"); + if (!data->statsFile) { + psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved); + psFree(resolved); + psFree(data); + return NULL; + } + psFree(resolved); + } + + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); + if (!data->in) { + data->in = psArrayAlloc(filenames->n); + } + if (!data->files) { + data->files = psArrayAlloc(filenames->n); + } + int numGood = 0; // Number of good files + for (int i = 0; i < filenames->n; i++) { + psString name = filenames->data[i]; // The name of the file + if (!name || strlen(name) == 0) { + continue; + } + psTrace("ppMerge", 1, "Checking input file %s....\n", name); + psString resolved = pmConfigConvertFilename(name, config, false); // Resolved file name + psFits *inFile = psFitsOpen(resolved, "r"); // The FITS file to read + if (!inFile) { + psLogMsg(__func__, PS_LOG_WARN, "Unable to open input file %s --- ignored.\n", resolved); + // Kick it out + psFree(filenames->data[i]); + filenames->data[i] = NULL; + continue; + } + psFree(resolved); + psMetadata *header = psFitsReadHeader(NULL, inFile); // The FITS (primary) header + data->files->data[i] = inFile; + + // The formats must be identical. The chief reason for this is so that we know what output format to + // use. I guess one could specify a different output format on the command line, but how do we + // generate a PHU for that? Perhaps we could revisit this restriction in the future (construct an + // FPAview from the specified camera format configuration, and use pmFPAAddSourceFromView), but for + // now it's less hassle just to limit the output format to be the input format. + if (!options->format) { + options->format = pmConfigCameraFormatFromHeader(config, header, true); + psFree(header); + if (!options->format) { + psLogMsg(__func__, PS_LOG_WARN, "Unable to identify camera format for input file %s --- " + "ignored.\n", name); + // Kick it out + psFree(header); + data->in->data[i] = NULL; + continue; + } + } else { + bool valid = false; + if (!pmConfigValidateCameraFormat(&valid, options->format, header)) { + psError (PS_ERR_UNKNOWN, false, "Error in config scripts\n"); + exit (PS_EXIT_CONFIG_ERROR); + } + if (!valid) { + psLogMsg(__func__, PS_LOG_WARN, "Input file %s doesn't match camera format --- ignored.\n", name); + // Kick it out + psFree(header); + data->in->data[i] = NULL; + continue; + } + } + + pmFPA *fpa = pmFPAConstruct(config->camera); + pmFPAview *view = pmFPAAddSourceFromHeader(fpa, header, options->format); + psFree(view); + + // Cull chips and cells that don't have data + // Otherwise the abundance of metadata in the concepts (esp. for GPC) can overload the memory + psArray *chips = fpa->chips; // Array of chips in output + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + psArray *cells = chip->cells; // Array of cells + int culled = 0; // Number of culled cells + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; + pmHDU *hdu = pmHDUGetLowest(fpa, chip, cell); // HDU for cell + if (!hdu || hdu->blankPHU) { + psFree(cell->concepts); + cell->concepts = NULL; + culled++; + } + } + if (culled == cells->n) { + psFree(chip->concepts); + chip->concepts = NULL; + } + } + data->in->data[i] = fpa; + + + // Use the first valid input as the basis for the output --- including the header + if (!data->out) { + psTrace("ppMerge", 5, "Constructing output using %s as a template.\n", name); + data->out = pmFPAConstruct(config->camera); + pmFPAview *view = pmFPAAddSourceFromHeader(data->out, header, options->format); + psFree(view); + } + psFree(header); + + psTrace("ppMerge", 3, "%s checks out.\n", name); + numGood++; + } + + // Count the cells + int numCells = 0; // Number of cells in the output FPA + psArray *chips = data->out->chips; // Array of chips in output + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + if (!chip) { + continue; + } + psArray *cells = chip->cells; // Array of cells + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; + if (cell) { + numCells++; + } + } + } + data->numCells = numCells; + psTrace("ppMerge", 3, "Output has %d cells.\n", numCells); + + psTrace("ppMerge", 3, "We have %d good inputs.\n", numGood); + if (numGood > 1) { + return data; + } + + psFree(data); + return NULL; +} + + Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeCheckInputs.h =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeCheckInputs.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeCheckInputs.h (revision 22293) @@ -0,0 +1,13 @@ +#ifndef PP_MERGE_CHECK_INPUTS_H +#define PP_MERGE_CHECK_INPUTS_H + +#include +#include "ppMergeOptions.h" +#include "ppMergeData.h" + +// Check input files to make sure everything's consistent +ppMergeData *ppMergeCheckInputs(ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +#endif Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeCombine.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeCombine.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeCombine.c (revision 22293) @@ -0,0 +1,434 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeCombine.h" +#include "ppMergeVersion.h" + +#define TESTING + + +#if 0 +static FILE *dumpFile = NULL; + +static psMemId mbAlloc(psMemBlock *mb) +{ + if (!dumpFile) { + dumpFile = fopen("memBlocks.dat", "w"); + } + fprintf(dumpFile, "Alloc: %12lu\t%12zd\t%s:%d\n", mb->id, mb->userMemorySize, + mb->file, mb->lineno); + return 1; +} + +static void dumpDone(void) +{ + fclose(dumpFile); + exit(EXIT_FAILURE); +} +#endif + +#if 0 +static psMemId memId = 0; +static void memDump(void) +{ + psMemBlock **leaks = NULL; + int numLeaks = psMemCheckLeaks(memId, &leaks, NULL, true); + FILE *memFile = fopen("mem.dat", "w"); + fprintf(memFile, "# MemBlock Size Source\n"); + for (int i = 0; i < numLeaks; i++) { + psMemBlock *mb = leaks[i]; + fprintf(memFile, "%12lu\t%12zd\t%s:%d\n", mb->id, mb->userMemorySize, + mb->file, mb->lineno); + } + fclose(memFile); + psFree(leaks); +} +#endif + +#if 0 +static void memCheck(void) +{ + return; + if (psTraceGetLevel("ppMerge") > 9) { + psMemBlock **leaks = NULL; + int numLeaks = psMemCheckLeaks(0, &leaks, NULL, true); + size_t largestSize = 0; + psMemId largest = 0; + size_t totalSize = 0; + for (int i = 0; i < numLeaks; i++) { + psMemBlock *mb = leaks[i]; + totalSize += mb->userMemorySize; + if (mb->userMemorySize > largestSize) { + largestSize = mb->userMemorySize; + largest = mb->id; + } + } + psFree(leaks); + psTrace("ppMerge", 0, "Memory in use: %zd\n", totalSize); + psTrace("ppMerge", 0, "Largest block: %ld\n", largest); + psTrace("ppMerge", 0, "sbrk(): %p\n", sbrk(0)); + } + return; +} +#endif + + +// Combine the inputs +bool ppMergeCombine(psImage *scales, // Scales for each cell of each integration, or NULL + psImage *zeros, // Zeros for each cell of each integration, or NULL + psArray *shutters, // Shutter correction data for each cell, or NULL + ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + // Sanity checks + assert(!options->scale || scales); + assert(!scales || (scales->type.type == PS_TYPE_F32 && + scales->numCols == data->numCells && + scales->numRows == filenames->n)); + assert(!options->zero || zeros); + assert(!zeros || (zeros->type.type == PS_TYPE_F32 && + zeros->numCols == data->numCells && + zeros->numRows == filenames->n)); + assert(!options->shutter || shutters); + assert(!shutters || (shutters->n == data->numCells)); + + // Iterate over the FPA + pmFPA *fpa = data->out; // Output FPA + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + int cellNum = -1; // Cell number in the whole FPA + if (data->out->hdu) { + pmFPAUpdateNames(data->out, NULL, NULL); + } + pmFPAWrite(data->out, data->outFile, config->database, true, false); // Write header only + pmChip *chip; // Chip of interest + psRandom *rng = NULL; // Random number generator; required for building a mask + pmHDU *lastHDU = NULL; // Last HDU to be updated + if (options->mask) { + rng = psRandomAlloc(PS_RANDOM_TAUS, 0); + } + while ((chip = pmFPAviewNextChip(view, fpa, 1))) { + if (chip->hdu) { + // Data will exist soon + pmFPAUpdateNames(data->out, chip, NULL); + chip->data_exists = true; + } + pmChipWrite(chip, data->outFile, config->database, true, false); // Write header only + pmCell *cell; // Cell of interest + while ((cell = pmFPAviewNextCell(view, fpa, 1))) { + cellNum++; + + pmHDU *hdu = pmHDUGetLowest(data->out, chip, cell); // HDU for cell + if (!hdu || hdu->blankPHU) { + pmCellWrite(cell, data->outFile, config->database, true); // Write header only + continue; + } + if (cell->hdu) { + // Data will exist soon + pmFPAUpdateNames(data->out, chip, cell); + chip->data_exists = cell->data_exists = true; + } + pmReadout *readout = pmReadoutAlloc(cell); // Output readout of interest + psArray *stack = psArrayAlloc(filenames->n); // Stack of readouts to combine + psVector *cellScales = NULL; // Scales for this cell + if (scales) { + cellScales = psImageCol(NULL, scales, cellNum); + } + psVector *cellZeros = NULL; // Zeros for this cell + if (zeros) { + cellZeros = psImageCol(NULL, zeros, cellNum); + } + + // Read bit by bit + int numRead; // Number of inputs read + int numScan = 0; + + // Put version metadata into header + if (hdu && hdu != lastHDU) { + if (!hdu->header) { + hdu->header = psMetadataAlloc(); + } + ppMergeVersionMetadata(hdu->header); + lastHDU = hdu; + } + + float shutterRef = NAN; // Reference shutter correction + if (options->shutter) { + shutterRef = pmShutterCorrectionReference(shutters->data[cellNum]); + } + + do { + numRead = 0; + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + psFits *fits = data->files->data[i]; // FITS file handle + if (!fits) { + continue; + } + + if (!stack->data[i]) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + pmChip *chipIn = fpaIn->chips->data[view->chip]; // Input chip + pmCell *cellIn = chipIn->cells->data[view->cell]; // Input cell + stack->data[i] = pmReadoutAlloc(cellIn); // Input readout + } + + // Only reading and writing the first readout in each cell (plane 0) + bool readOK; + if (pmReadoutReadNext(&readOK, stack->data[i], fits, 0, options->rows)) { + if (!readOK) { + psError(PS_ERR_IO, false, "Failed to read concepts for cell.\n"); + psErrorStackPrint(stderr, "trouble reading data!\n"); + exit (1); + } + // If we're creating a bias or a dark, we don't want to generate a mask + if ((options->zero || options->scale || options->shutter || options->mask) && + options->combine->maskVal) { + pmReadoutSetMask(stack->data[i], options->satMask, options->badMask); + } + + // If we're combining with weights, we want to generate weights. + if (options->combine->weights && !options->dark) { + + // If it's a bias or dark, set the gain to zero: noise only contributed by read + if (!options->zero && !options->scale) { + pmReadoutSetWeight(stack->data[i], false); + } else { + pmReadoutSetWeight(stack->data[i], true); + } + } + + numRead++; + } else { + psTrace("ppMerge", 3, "Finished reading file %d, chip %d, cell %d, scan %d\n", + i, view->chip, view->cell, numScan); + } + + } + + psTrace("ppMerge", 5, "Chip %d, cell %d, scan %d\n", view->chip, view->cell, numScan); + if (numRead > 0) { + if (options->shutter) { + pmShutterCorrectionGenerate(readout, NULL, stack, shutterRef, shutters->data[cellNum], + options->shutterIter, options->shutterRej, + options->combine->maskVal); + } else if (options->dark) { + pmDarkCombine(cell, stack, options->darkOrdinates, options->darkNorm, + options->combine->iter, options->combine->rej, + options->combine->maskVal); + } else { + pmReadoutCombine(readout, stack, cellZeros, cellScales, options->combine); + } + } + numScan++; + + } while (numRead > 0); + + // Get list of cells for concepts averaging + psList *inCells = psListAlloc(NULL); // List of cells + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmCell *cellIn = pmFPAviewThisCell(view, data->in->data[i]); // Input cell + psListAdd(inCells, PS_LIST_TAIL, cellIn); + } + if (!pmConceptsAverageCells(cell, inCells, NULL, NULL, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average cell concepts."); + psFree(inCells); + return false; + } + psFree(inCells); + + psFree(stack); + +#if 0 + // Set the dark time for the output image, since we normalised + if (options->darktime) { + psMetadataItem *darkItem = psMetadataLookup(cell->concepts, "CELL.DARKTIME"); + darkItem->data.F32 = 1.0; + psMetadataItem *expItem = psMetadataLookup(cell->concepts, "CELL.EXPOSURE"); + expItem->data.F32 = 1.0; + } +#endif + + // Measure the fringes for this cell + // + // XXX Need to deal with multiple components: we will do this by building up the components + // Read the existing fringe measurements + // Use existing regions to measure fringe statistics + // Add the new fringe measurements to the existing fringe measurements. + // Write the appended fringe measurements. + // Read in the "output" file to get the existing components. + // Put the new readout into the cell after the existing readouts. + if (options->fringe && readout->image) { + pmFringeRegions *regions = pmFringeRegionsAlloc(options->fringeNum, options->fringeSize, + options->fringeSize, options->fringeSmoothX, + options->fringeSmoothY); // Fringe regions + pmFringeStats *fringe = pmFringeStatsMeasure(regions, readout, options->combine->maskVal); + psFree(regions); + if (!fringe) { + psError(PS_ERR_UNKNOWN, false, "Unable to measure fringe statistics.\n"); + psFree(readout); + return false; + } + + psArray *fringes = psArrayAlloc(1); // Array of fringes + fringes->data[0] = fringe; + + pmFringesFormat(cell, NULL, fringes); + psFree(fringes); // Drop reference + } + + if (readout->image) { + // Add MD5 information for cell + pmHDU *hdu = pmHDUFromCell(cell); // HDU that owns the cell + const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME"); + const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME"); + + psString headerName = NULL; // Header name for MD5 + psStringAppend(&headerName, "MD5_%s_%s", chipName, cellName); + + psVector *md5 = psImageMD5(readout->image); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, headerName, PS_META_REPLACE, + "Image MD5", md5string); + psFree(md5string); + psFree(headerName); + + + // Statistics on the merged cell + if (data->statsFile) { + if (!data->stats) { + data->stats = psMetadataAlloc(); + } + if (!ppStatsFPA(data->stats, data->out, view, + options->combine->maskVal | pmConfigMask("BLANK", config), + config)) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to generate stats for image.\n"); + return false; + } + } + } + + psFree(readout); // Drop reference + + + // We threw away the bias sections --- record this + psMetadataItem *biassecItem = psMetadataLookup(cell->concepts, "CELL.BIASSEC"); // Item of BIASSEC + psList *biassecList = biassecItem->data.V; // List of BIASSECs + while (psListRemove(biassecList, PS_LIST_TAIL)); // Removing all entries + + // Blow away the cell data + for (int i = 0; i < filenames->n; i++) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + if (!fpaIn) { continue; } // was not a valid input file + pmChip *chipIn = fpaIn->chips->data[view->chip]; // Input chip + pmCell *cellIn = chipIn->cells->data[view->cell]; // Input cell + pmCellFreeData(cellIn); + } + + // Write the pixels + if (cell->hdu && !cell->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out cell HDU.\n"); + if (options->dark) { + pmCellWriteDark(cell, data->outFile, config->database, false); + } else { + pmCellWrite(cell, data->outFile, config->database, false); + if (options->fringe) { + pmCellWriteTable(data->outFile, cell, "FRINGE"); + } + } + pmCellFreeData(cell); + } + } + + // Blow away the chip data + for (int i = 0; i < filenames->n; i++) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + if (!fpaIn) { continue; } // was not a valid input file + pmChip *chipIn = fpaIn->chips->data[view->chip]; // Input chip + pmChipFreeData(chipIn); + } + + // Write the pixels + if (chip->hdu && !chip->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out chip HDU.\n"); + if (options->dark) { + pmChipWriteDark(chip, data->outFile, config->database, false, false); + } else { + pmChipWrite(chip, data->outFile, config->database, false, false); + if (options->fringe) { + pmChipWriteTable(data->outFile, chip, "FRINGE"); + } + } + + pmChipFreeData(chip); + } + } + + // Blow away the FPA data + for (int i = 0; i < filenames->n; i++) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + if (!fpaIn) { continue; } // was not a valid input file + pmFPAFreeData(fpaIn); + } + + if (data->out->hdu && !data->out->hdu->blankPHU) { + // Write the pixels + psTrace("ppMerge", 5, "Writing out FPA HDU.\n"); + if (options->dark) { + pmFPAWriteDark(data->out, data->outFile, config->database, false, false); + } else { + pmFPAWrite(data->out, data->outFile, config->database, false, false); + if (options->fringe) { + pmFPAWriteTable(data->outFile, fpa, "FRINGE"); + } + } + } + + pmFPAFreeData(data->out); + + psFree(view); + psFree(rng); + + + // Get list of FPAs for concepts averaging + psList *inFPAs = psListAlloc(NULL); // List of FPAs + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmFPA *fpaIn = data->in->data[i]; // Input FPA + psListAdd(inFPAs, PS_LIST_TAIL, fpaIn); + } + + if (!pmConceptsAverageFPAs(fpa, inFPAs)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts."); + psFree(inFPAs); + return false; + } + psFree(inFPAs); + + + return true; + +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeCombine.h =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeCombine.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeCombine.h (revision 22293) @@ -0,0 +1,19 @@ +#ifndef PP_MERGE_COMBINE_H +#define PP_MERGE_COMBINE_H + +#include +#include + +#include "ppMergeData.h" +#include "ppMergeOptions.h" + +// Combine readouts +bool ppMergeCombine(psImage *scales, // Scales for each cell of each integration, or NULL + psImage *zeros, // Zeros for each cell of each integration, or NULL + psArray *shutters, // Shutter correction data for each cell, or NULL + ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +#endif Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeConfig.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeConfig.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeConfig.c (revision 22293) @@ -0,0 +1,93 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeConfig.h" + +#define DONT_USE_DB + +// Output usage information +static void usage(const char *programName, // Name of the program + psMetadata *arguments // Arguments list + ) +{ + printf("Merge multiple calibration frames into a master frame by stacking.\n\n" + "Usage:\n" + "\t%s OUTPUT.fits INPUT1.fits INPUT2.fits ...\n" + "\n", programName); + psArgumentHelp(arguments); + psFree(arguments); + exit(EXIT_FAILURE); +} + +pmConfig *ppMergeConfig(int argc, char **argv) +{ + pmConfig *config = pmConfigRead(&argc, argv, PPMERGE_RECIPE); + // Load the site-wide configuration information + if (! config) { + psErrorStackPrint(stderr, "Can't find site configuration!\n"); + exit(EXIT_FAILURE); + } + + psMetadata *arguments = psMetadataAlloc(); // Command-line arguments + psMetadataAddStr(arguments, PS_LIST_TAIL, "-type", 0, "Type of calibration frame", ""); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-zero", 0, "Subtract background?", false); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-scale", 0, "Scale by background?", false); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-exptime", 0, "Scale by the exposure time?", false); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-onoff", 0, "Number of on/off pairs", 0); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-stats", 0, "MDC file to hold statistics ", NULL); + + if (argc == 1) { + psFree(config); + usage(argv[0], arguments); + } + + // Parse the arguments + if (!psArgumentParse(arguments, &argc, argv) || argc < 3) { + psErrorStackPrint(stderr, "Unable to parse arguments."); + psFree(config); + usage(argv[0], arguments); + } + + // Add the output image to the arguments list + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", + argv[1]); + + psMetadataCopy(config->arguments, arguments); + psFree(arguments); + + // Everything remaining must be input files + if (argc - 2 <= 1) { + psErrorStackPrint(stderr, "No files to combine.\n"); + exit(EXIT_FAILURE); + } + psArray *files = psArrayAlloc(argc - 2); + for (int i = 2; i < argc; i++) { + files->data[i - 2] = psStringCopy(argv[i]); + } + psMetadataAddPtr(config->arguments, PS_LIST_TAIL, "INPUT", PS_DATA_ARRAY, + "Array of inputs images", files); + psFree(files); // Drop reference + +#ifndef DONT_USE_DB +#ifdef HAVE_PSDB + // Define database handle, if required + config->database = pmConfigDB(config); +#endif +#endif + + // Add concepts for scale and zero + psMetadataItem *scaleItem = psMetadataItemAllocF32("PPMERGE.SCALE", "Scaling for ppMerge", NAN); + psMetadataItem *zeroItem = psMetadataItemAllocF32("PPMERGE.ZERO", "Zero offset for ppMerge", NAN); + pmConceptRegister(scaleItem, NULL, NULL, false, PM_FPA_LEVEL_CELL); + pmConceptRegister(zeroItem, NULL, NULL, false, PM_FPA_LEVEL_CELL); + psFree(scaleItem); + psFree(zeroItem); + + return config; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeConfig.h =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeConfig.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeConfig.h (revision 22293) @@ -0,0 +1,10 @@ +#ifndef PP_MERGE_CONFIG_H +#define PP_MERGE_CONFIG_H + +#include + +// Get the configuration information +pmConfig *ppMergeConfig(int argc, char **argv // The standard command-line parameters (but pointer to number) + ); + +#endif Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeData.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeData.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeData.c (revision 22293) @@ -0,0 +1,50 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" + +// Free function for ppMergeData +static void mergeDataFree(ppMergeData *data // Data to free + ) +{ + if (data->files) { + for (long i = 0; i < data->files->n; i++) { + psFitsClose(data->files->data[i]); + data->files->data[i] = NULL; + } + psFree(data->files); + } + psFree(data->in); + psFree(data->out); + if (data->outFile) { + psFitsClose(data->outFile); + data->outFile = NULL; + } + if (data->statsFile) { + fclose(data->statsFile); + data->statsFile = NULL; + } + psFree(data->stats); +} + +// Allocator for ppMergeData +ppMergeData *ppMergeDataAlloc(void) +{ + ppMergeData *data = psAlloc(sizeof(ppMergeData)); // The data, to return + psMemSetDeallocator(data, (psFreeFunc)mergeDataFree); + + data->numCells = 0; + data->files = NULL; + data->in = NULL; + data->out = NULL; + data->outFile = NULL; + data->stats = NULL; + data->statsFile = NULL; + + return data; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeData.h =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeData.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeData.h (revision 22293) @@ -0,0 +1,23 @@ +#ifndef PP_MERGE_DATA_H +#define PP_MERGE_DATA_H + +#include +#include + +// Container for the data +typedef struct { + int numCells; // Number of (valid) cells in the FPA + psArray *files; // Input file pointers + psArray *in; // Input FPA structures + pmFPA *out; // Output FPA structure + psFits *outFile; // FITS file handle for output + psMetadata *stats; // Statistics on the combined image + FILE *statsFile; // File stream for statistics output +} ppMergeData; + + +// Allocator +ppMergeData *ppMergeDataAlloc(void); + + +#endif Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeErrorCodes.c.in =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeErrorCodes.c.in (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeErrorCodes.c.in (revision 22293) @@ -0,0 +1,26 @@ +/* + * The line + { PPMERGE_ERR_$X{ErrorCode}, "$X{ErrorDescription}"}, + * (without the Xs) + * will be replaced by values from errorCodes.dat + */ +#include "pslib.h" +#include "ppMergeErrorCodes.h" + +void ppMergeErrorRegister(void) +{ + static psErrorDescription errors[] = { + { PPMERGE_ERR_BASE, "First value we use; lower values belong to psLib" }, + { PPMERGE_ERR_${ErrorCode}, "${ErrorDescription}"}, + }; + static int nerror = PPMERGE_ERR_NERROR - PPMERGE_ERR_BASE; // number of values in enum + + for (int i = 0; i < nerror; i++) { + psErrorDescription *tmp = psAlloc(sizeof(psErrorDescription)); + p_psMemSetPersistent(tmp, true); + *tmp = errors[i]; + psErrorRegister(tmp, 1); + psFree(tmp); /* it's on the internal list */ + } + nerror = 0; // don't register more than once +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeErrorCodes.dat =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeErrorCodes.dat (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeErrorCodes.dat (revision 22293) @@ -0,0 +1,10 @@ +# +# This file is used to generate ppMergeErrorClasses.h +# +BASE = 700 First value we use; lower values belong to psLib +# these errors correspond to standard exit conditions +ARGUMENTS Incorrect arguments +SYS System error +CONFIG Problem in configure files +PROG Programming error +DATA invalid data Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeErrorCodes.h.in =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeErrorCodes.h.in (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeErrorCodes.h.in (revision 22293) @@ -0,0 +1,18 @@ +#if !defined(PPMERGE_ERROR_CODES_H) +#define PPMERGE_ERROR_CODES_H +/* + * The line + * PPMERGE_ERR_$X{ErrorCode}, + * (without the X) + * + * will be replaced by values from errorCodes.dat + */ +typedef enum { + PPMERGE_ERR_BASE = 512, + PPMERGE_ERR_${ErrorCode}, + PPMERGE_ERR_NERROR +} ppMergeErrorCode; + +void ppMergeErrorRegister(void); + +#endif Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMask.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMask.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMask.c (revision 22293) @@ -0,0 +1,214 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask +bool ppMergeMask(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + for (int i = 0; i < 2; i++) { + if (!ppMergeMaskSuspect (data, options, config)) { + psError(PS_ERR_UNKNOWN, true, "failed on mask suspect.\n"); + return false; + } + + if (!ppMergeMaskBad (data, options, config)) { + psError(PS_ERR_UNKNOWN, true, "failed on mask bad.\n"); + return false; + } + } + + if (!ppMergeMaskAverageConcepts (data, options, config)) { + psError(PS_ERR_UNKNOWN, true, "failed on average concepts.\n"); + return false; + } + + if (!ppMergeMaskGrow (data, options, config)) { + psError(PS_ERR_UNKNOWN, true, "failed on mask grow.\n"); + return false; + } + + if (!ppMergeMaskWrite (data, options, config)) { + psError(PS_ERR_UNKNOWN, true, "failed on mask write.\n"); + return false; + } + + return true; +} + +bool ppMergeMaskReadoutStats(const pmReadout *readout, + const pmReadout *output, + ppMergeOptions *options, // Options + psRandom *rng) +{ + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_IMAGE_NON_NULL(readout->image, false); + PS_ASSERT_IMAGE_NON_EMPTY(readout->image, false); + PS_ASSERT_IMAGE_TYPE(readout->image, PS_TYPE_F32, false); + if (readout->mask) { + PS_ASSERT_IMAGE_NON_EMPTY(readout->mask, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(readout->image, readout->mask, false); + PS_ASSERT_IMAGE_TYPE(readout->mask, PS_TYPE_MASK, false); + } + + psImage *mask = NULL; + psImage *image = readout->image; // Image of interest + + if (output) { + mask = output->mask; // Corresponding mask + } + + if (rng) { + psMemIncrRefCounter(rng); + } else { + rng = psRandomAlloc(PS_RANDOM_TAUS, 0); + } + + // XXX note that this now will accept any of several stats options + psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + stats->nSubsample = options->sample; + + if (!psImageBackground(stats, NULL, image, mask, options->combine->maskVal, rng)) { + psError(PS_ERR_UNKNOWN, false, "Failure to measure image statistics.\n"); + psFree(stats); + psFree(rng); + return false; + } + if (!isfinite(stats->robustMedian) || !isfinite(stats->robustUQ) || !isfinite(stats->robustLQ)) { + psError(PS_ERR_UNKNOWN, false, "invalide image statistics (nan).\n"); + psFree(stats); + psFree(rng); + return false; + } + + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "READOUT.MEDIAN", PS_META_REPLACE, "image stats", stats->robustMedian); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "READOUT.STDEV", PS_META_REPLACE, "image stats", stats->robustStdev); + + psFree(rng); + return true; +} + +bool ppMergeMaskChipStats (const pmChip *chip, + const pmChip *output, + ppMergeOptions *options, + psRandom *rng) +{ + PS_ASSERT_PTR_NON_NULL(chip, false); + + // XXX note that this now will accept any of several stats options + psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + + if (rng) { + psMemIncrRefCounter(rng); + } else { + rng = psRandomAlloc(PS_RANDOM_TAUS, 0); + } + + // accumulate a vector of data values using options->sample per readout + psVector *values = psVectorAllocEmpty(options->sample, PS_TYPE_F32); // Vector containing subsample + + for (int nCell = 0; nCell < chip->cells->n; nCell++) { + pmCell *cell = chip->cells->data[nCell]; + if (cell->readouts->n == 0) continue; + + for (int nReadout = 0; nReadout < cell->readouts->n; nReadout++) { + pmReadout *readout = cell->readouts->data[nReadout]; + if (!readout->image) continue; + + psTrace("ppMerge", 4, "Measure statistics for cell %d, readout %d\n", nCell, nReadout); + + psImage *image = readout->image; // Image of interest + + pmCell *cellOutput = output->cells->data[nCell]; + pmReadout *readoutOutput = NULL; + psImage *mask = NULL; + if (cellOutput->readouts->n > 0) { + readoutOutput = cellOutput->readouts->data[0]; + mask = readoutOutput->mask; // Corresponding mask + } + + // Size of image + long nx = image->numCols; + long ny = image->numRows; + const int Npixels = nx*ny; // Total number of pixels + const int Nsubset = PS_MIN(options->sample, Npixels); // Number of pixels in subset + + values = psVectorRealloc (values, values->n + Nsubset); // make sure we have enough space + + // select a subset of the image pixels to measure the stats + for (long i = 0; i < Nsubset; i++) { + double frnd = psRandomUniform(rng); + int pixel = Npixels * frnd; + int ix = pixel % nx; + int iy = pixel / nx; + + if (!isfinite(image->data.F32[iy][ix])) continue; + if (mask && (mask->data.U8[iy][ix] & options->combine->maskVal)) continue; + + float value = image->data.F32[iy][ix]; + values->data.F32[values->n] = value; + values->n ++; + } + } + } + + // no valid data, skip the chip + if (!values->n) { + psFree(values); + psFree(stats); + psFree(rng); + return true; + } + + // calculate the statistics + if (!psVectorStats (stats, values, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "Unable to measure statistics for chip"); + psFree(values); + psFree(stats); + psFree(rng); + return false; + } + if (!isfinite(stats->robustMedian) || !isfinite(stats->robustUQ) || !isfinite(stats->robustLQ)) { + psError(PS_ERR_UNKNOWN, false, "invalid image statistics (nan).\n"); + psFree(values); + psFree(stats); + psFree(rng); + return false; + } + + // supply the stats to the readout analysis metadata + for (int nCell = 0; nCell < chip->cells->n; nCell++) { + pmCell *cell = chip->cells->data[nCell]; + if (cell->readouts->n == 0) continue; + + for (int nReadout = 0; nReadout < cell->readouts->n; nReadout++) { + pmReadout *readout = cell->readouts->data[nReadout]; + if (!readout->image) continue; + + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "READOUT.MEDIAN", PS_META_REPLACE, "image stats", stats->robustMedian); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "READOUT.STDEV", PS_META_REPLACE, "image stats", stats->robustStdev); + } + } + + psLogMsg ("ppMerge", PS_LOG_INFO, "statistics for chip: %f +/- %f\n", stats->robustMedian, stats->robustStdev); + + psFree(values); + psFree(stats); + psFree(rng); + return true; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMask.h =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMask.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMask.h (revision 22293) @@ -0,0 +1,50 @@ +#ifndef PP_MERGE_MASK +#define PP_MERGE_MASK + +#include +#include "ppMergeData.h" +#include "ppMergeOptions.h" + +// Generate a mask +bool ppMergeMask(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +bool ppMergeMaskSuspect(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +bool ppMergeMaskBad(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +bool ppMergeMaskAverageConcepts(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +bool ppMergeMaskWrite(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +bool ppMergeMaskGrow(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +bool ppMergeMaskGrowReadout (pmReadout *readout, psMaskType maskVal, int nPixels); + +bool ppMergeMaskReadoutStats(const pmReadout *readout, + const pmReadout *output, + ppMergeOptions *options, // Options + psRandom *rng); + +bool ppMergeMaskChipStats (const pmChip *chip, + const pmChip *output, + ppMergeOptions *options, + psRandom *rng); +#endif Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskAverageConcepts.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskAverageConcepts.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskAverageConcepts.c (revision 22293) @@ -0,0 +1,83 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask +bool ppMergeMaskAverageConcepts(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + pmFPA *fpaOut = data->out; // Output FPA + + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + if (fpaOut->hdu) { + pmFPAUpdateNames(fpaOut, NULL, NULL); + } + + pmChip *chipOut; // Output chip of interest + while ((chipOut = pmFPAviewNextChip(view, fpaOut, 1))) { + + pmCell *cellOut; // Output cell of interest + while ((cellOut = pmFPAviewNextCell(view, fpaOut, 1))) { + + pmHDU *hdu = pmHDUGetLowest(fpaOut, chipOut, cellOut); + if (!hdu || hdu->blankPHU) { + continue; + } + + // Get list of cells for concepts averaging + psList *inCells = psListAlloc(NULL); // List of cells + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmCell *cellIn = pmFPAviewThisCell(view, data->in->data[i]); // Input cell + psListAdd(inCells, PS_LIST_TAIL, cellIn); + } + if (!pmConceptsAverageCells(cellOut, inCells, NULL, NULL, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average cell concepts."); + psFree(inCells); + return false; + } + psFree(inCells); + } + } + + // Get list of FPAs for concepts averaging + psList *inFPAs = psListAlloc(NULL); // List of FPAs + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmFPA *fpaIn = data->in->data[i]; // Input FPA + psListAdd(inFPAs, PS_LIST_TAIL, fpaIn); + } + + if (!pmConceptsAverageFPAs(fpaOut, inFPAs)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts."); + psFree(inFPAs); + return false; + } + psFree(inFPAs); + + psFree(view); + + return true; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskBad.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskBad.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskBad.c (revision 22293) @@ -0,0 +1,83 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask +bool ppMergeMaskBad(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + pmFPA *fpaOut = data->out; // Output FPA + + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + + pmChip *chipOut; // Output chip of interest + while ((chipOut = pmFPAviewNextChip(view, fpaOut, 1))) { + + pmCell *cellOut; // Output cell of interest + while ((cellOut = pmFPAviewNextCell(view, fpaOut, 1))) { + + psImage *suspect = psMetadataLookupPtr(NULL, cellOut->analysis, "MASK.SUSPECT"); + if (! suspect) { + continue; + } + + pmReadout *roOut = NULL; + if (cellOut->readouts->n > 0) { + roOut = psMemIncrRefCounter (cellOut->readouts->data[0]); + psFree (roOut->mask); + psTrace("ppMerge", 5, "save results with mask from previous pass\n"); + } else { + roOut = pmReadoutAlloc(cellOut); // Output readout + } + + roOut->mask = pmMaskIdentifyBadPixels(suspect, options->combine->maskVal, filenames->n, options->maskBad, options->maskMode); + roOut->data_exists = cellOut->data_exists = chipOut->data_exists = true; + + // Statistics on the merged cell + if (data->statsFile) { + if (!data->stats) { + data->stats = psMetadataAlloc(); + } + + // Build a fake image to do statistics + roOut->image = psImageAlloc(roOut->mask->numCols, roOut->mask->numRows, PS_TYPE_F32); + psImageInit(roOut->image, 1.0); + if (!ppStatsFPA(data->stats, data->out, view, + options->combine->maskVal | pmConfigMask("BLANK", config), + config)) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to generate stats for image.\n"); + return false; + } + psFree(roOut->image); + roOut->image = NULL; + } + psFree(roOut); // Drop reference + + // drop this reference (unless??) + psMetadataRemoveKey (cellOut->analysis, "MASK.SUSPECT"); + } + } + + psFree(view); + + return true; +} + Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskByImageStats.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskByImageStats.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskByImageStats.c (revision 22293) @@ -0,0 +1,87 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask image consisting of pixels which are outliers from the image mean +bool ppMergeMaskByImageStats(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator + pmFPA *fpaOut = data->out; // Output FPA + + // Iterate over each file + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + psFits *fits = data->files->data[i]; // FITS file handle + if (!fits) { + continue; + } + psTrace("ppMerge", 3, "File %d: %s\n", i, (const char*)filenames->data[i]); + + pmFPA *fpaIn = data->in->data[i]; // Input FPA + pmFPAview *view = pmFPAviewAlloc(0); // View of FPA, for iteration + pmChip *chipIn; // Input chip of interest + while ((chipIn = pmFPAviewNextChip(view, fpaIn, 1))) { + pmCell *cellIn; // Input cell of interest + while ((cellIn = pmFPAviewNextCell(view, fpaIn, 1))) { + if (!pmCellRead(cellIn, fits, config->database)) { + continue; + } + if (cellIn->readouts->n == 0) { + continue; + } + + pmCell *cellOut = pmFPAviewThisCell(view, fpaOut); // Output cell + // Suspect pixels image + psImage *suspect = psMetadataLookupPtr(NULL, cellOut->analysis, "MASK.SUSPECT"); + bool first = suspect ? false : true; + + pmReadout *roIn; // Input readout of interest + while ((roIn = pmFPAviewNextReadout(view, fpaIn, 1))) { + if (!roIn->image) { + continue; + } + psTrace("ppMerge", 4, "Flagging suspect pixels in chip %d, cell %d, ro %d\n", + view->chip, view->cell, view->readout); + float frac = options->sample / (float)(roIn->image->numCols * roIn->image->numRows); + suspect = pmMaskFlagSuspectPixels(suspect, roIn, options->maskSuspect, + options->combine->maskVal, frac, rng); + } + + if (first) { + psMetadataAddImage(cellOut->analysis, PS_LIST_TAIL, "MASK.SUSPECT", 0, + "Suspect pixels", suspect); + psFree(suspect); + } + + pmCellFreeData(cellIn); + } + pmChipFreeData(chipIn); + } + pmFPAFreeData(fpaIn); + psFree(view); + } + psFree(rng); + + return true; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskByPixelStats.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskByPixelStats.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskByPixelStats.c (revision 22293) @@ -0,0 +1,177 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeCombine.h" +#include "ppMergeVersion.h" + +// XXX for the moment, this function and ppMergeMaskByImageStats both load (and free) the input +// data + +// Combine the inputs +bool ppMergeMaskByPixelStats(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + // Iterate over the FPA + pmFPA *fpa = data->out; // Output FPA + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + int cellNum = -1; // Cell number in the whole FPA + if (data->out->hdu) { + pmFPAUpdateNames(data->out, NULL, NULL); + } + + pmChip *chip; // Chip of interest + pmHDU *lastHDU = NULL; // Last HDU to be updated + + while ((chip = pmFPAviewNextChip(view, fpa, 1))) { + if (chip->hdu) { + // Data will exist soon + pmFPAUpdateNames(data->out, chip, NULL); + chip->data_exists = true; + } + pmCell *cell; // Cell of interest + while ((cell = pmFPAviewNextCell(view, fpa, 1))) { + cellNum++; + if (cell->hdu) { + // Data will exist soon + pmFPAUpdateNames(data->out, chip, cell); + chip->data_exists = cell->data_exists = true; + } + pmReadout *readout = pmReadoutAlloc(cell); // Output readout of interest + psArray *stack = psArrayAlloc(filenames->n); // Stack of readouts to combine + + // Read bit by bit + int numRead; // Number of inputs read + int numScan = 0; + + // Put version metadata into header + pmHDU *hdu = pmHDUFromCell(cell); + if (hdu && hdu != lastHDU) { + if (!hdu->header) { + hdu->header = psMetadataAlloc(); + } + ppMergeVersionMetadata(hdu->header); + lastHDU = hdu; + } + + while (1) { + numRead = 0; + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + psFits *fits = data->files->data[i]; // FITS file handle + if (!fits) { + continue; + } + + if (!stack->data[i]) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + pmChip *chipIn = fpaIn->chips->data[view->chip]; // Input chip + pmCell *cellIn = chipIn->cells->data[view->cell]; // Input cell + stack->data[i] = pmReadoutAlloc(cellIn); // Input readout + } + + // Only reading and writing the first readout in each cell (plane 0) + bool readOK; + if (pmReadoutReadNext(&readOK, stack->data[i], fits, 0, options->rows)) { + if (!readOK) { + psError(PS_ERR_IO, false, "Failed to read concepts for cell.\n"); + psErrorStackPrint(stderr, "trouble reading data!\n"); + exit (1); + } + numRead++; + } else { + psTrace("ppMerge", 3, "Unable to read from file %d for chip %d, " + "cell %d, scan %d\n", i, view->chip, view->cell, numScan); + } + } + + psTrace("ppMerge", 5, "Chip %d, cell %d, scan %d\n", view->chip, view->cell, numScan); + if (numRead == 0) { + break; + } + ppMergeMaskReadoutByPixelStats(readout, stack, options->combine); + numScan++; + } + + // Get list of cells for concepts averaging + // XXX only do this operation once between Image and Pixel Stats + psList *inCells = psListAlloc(NULL); // List of cells + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmCell *cellIn = pmFPAviewThisCell(view, data->in->data[i]); // Input cell + psListAdd(inCells, PS_LIST_TAIL, cellIn); + } + if (!pmConceptsAverageCells(cell, inCells, NULL, NULL, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average cell concepts."); + psFree(inCells); + return false; + } + psFree(inCells); + psFree(stack); + psFree(readout); // Drop reference + + // Blow away the cell data + for (int i = 0; i < filenames->n; i++) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + if (!fpaIn) { continue; } // was not a valid input file + pmChip *chipIn = fpaIn->chips->data[view->chip]; // Input chip + pmCell *cellIn = chipIn->cells->data[view->cell]; // Input cell + pmCellFreeData(cellIn); + } + } + + // Blow away the chip data + for (int i = 0; i < filenames->n; i++) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + if (!fpaIn) { continue; } // was not a valid input file + pmChip *chipIn = fpaIn->chips->data[view->chip]; // Input chip + pmChipFreeData(chipIn); + } + } + + // Blow away the FPA data + for (int i = 0; i < filenames->n; i++) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + if (!fpaIn) { continue; } // was not a valid input file + pmFPAFreeData(fpaIn); + } + + psFree(view); + + // Get list of FPAs for concepts averaging + psList *inFPAs = psListAlloc(NULL); // List of FPAs + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmFPA *fpaIn = data->in->data[i]; // Input FPA + psListAdd(inFPAs, PS_LIST_TAIL, fpaIn); + } + if (!pmConceptsAverageFPAs(fpa, inFPAs)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts."); + psFree(inFPAs); + return false; + } + psFree(inFPAs); + + return true; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskGrow.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskGrow.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskGrow.c (revision 22293) @@ -0,0 +1,89 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask +bool ppMergeMaskGrow(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + pmFPA *fpaOut = data->out; // Output FPA + + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + + pmChip *chipOut; // Output chip of interest + while ((chipOut = pmFPAviewNextChip(view, fpaOut, 1))) { + + pmCell *cellOut; // Output cell of interest + while ((cellOut = pmFPAviewNextCell(view, fpaOut, 1))) { + + pmReadout *roOut; + while ((roOut = pmFPAviewNextReadout(view, fpaOut, 1))) { + + // do this N iterations (XXX add to options) + ppMergeMaskGrowReadout(roOut, options->combine->maskVal, options->growPixels); + ppMergeMaskGrowReadout(roOut, options->combine->maskVal, options->growPixels); + } + } + } + + psFree(view); + + return true; +} + +bool ppMergeMaskGrowReadout (pmReadout *readout, psMaskType maskVal, int nPixels) { + + if (!readout->mask) return true; + + psImage *oldMask = readout->mask; + psImage *newMask = psImageCopy (NULL, oldMask, PS_TYPE_U8); + + psImageInit (newMask, 0); + + int Nx = oldMask->numCols; + int Ny = oldMask->numRows; + + for (int iy = 0; iy < Ny; iy++) { + for (int ix = 0; ix < Nx; ix++) { + int nSum = 0; + for (int jy = -1; jy <= +1; jy++) { + int ny = iy + jy; + if (ny < 0) continue; + if (ny >= Ny) continue; + for (int jx = -1; jx <= +1; jx++) { + if (!jx && !jy) continue; + int nx = ix + jx; + if (nx < 0) continue; + if (nx >= Nx) continue; + if (oldMask->data.U8[ny][nx] & maskVal) { + nSum ++; + } + } + } + if (nSum >= nPixels) { + newMask->data.U8[iy][ix] = maskVal; + } else { + newMask->data.U8[iy][ix] = oldMask->data.U8[iy][ix]; + } + } + } + psFree (readout->mask); + readout->mask = newMask; + + return true; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskOutput.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskOutput.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskOutput.c (revision 22293) @@ -0,0 +1,157 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// write out the mask: assumes it has been saved on the cell->analysis as MASK.UNION +bool ppMergeMaskOutput(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + pmFPA *fpaOut = data->out; // Output FPA + + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + if (fpaOut->hdu) { + pmFPAUpdateNames(fpaOut, NULL, NULL); + } + pmFPAWriteMask(fpaOut, data->outFile, config->database, true, false); // Write header only + pmChip *chipOut; // Output chip of interest + while ((chipOut = pmFPAviewNextChip(view, fpaOut, 1))) { + if (chipOut->hdu) { + chipOut->data_exists = true; + pmFPAUpdateNames(fpaOut, chipOut, NULL); + } + pmChipWriteMask(chipOut, data->outFile, config->database, true, false); // Write header only + pmCell *cellOut; // Output cell of interest + while ((cellOut = pmFPAviewNextCell(view, fpaOut, 1))) { + if (cellOut->hdu) { + chipOut->data_exists = cellOut->data_exists = true; + pmFPAUpdateNames(fpaOut, chipOut, cellOut); + } + pmCellWriteMask(cellOut, data->outFile, config->database, true); // Write header only + + psImage *maskUnion = psMetadataLookupPtr(NULL, cellOut->analysis, "MASK.UNION"); + if (! maskUnion) { + continue; + } + + pmReadout *roOut = pmReadoutAlloc(cellOut); // Output readout + roOut->mask = maskUnion; + roOut->data_exists = cellOut->data_exists = chipOut->data_exists = true; + + // XXX skip this? Get list of cells for concepts averaging + { + psList *inCells = psListAlloc(NULL); // List of cells + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmCell *cellIn = pmFPAviewThisCell(view, data->in->data[i]); // Input cell + psListAdd(inCells, PS_LIST_TAIL, cellIn); + } + if (!pmConceptsAverageCells(cellOut, inCells, NULL, NULL, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average cell concepts."); + psFree(inCells); + return false; + } + psFree(inCells); + } + + { + // Add MD5 information for cell + pmHDU *hdu = pmHDUFromCell(cell); // HDU that owns the cell + const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME"); + const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME"); + + psString headerName = NULL; // Header name for MD5 + psStringAppend(&headerName, "MD5_%s_%s", chipName, cellName); + + psVector *md5 = psImageMD5(readout->image); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, headerName, PS_META_REPLACE, + "Image MD5", md5string); + psFree(md5string); + psFree(headerName); + } + + // Statistics on the merged cell + if (data->statsFile) { + if (!data->stats) { + data->stats = psMetadataAlloc(); + } + + // Build a fake image to do statistics + roOut->image = psImageAlloc(roOut->mask->numCols, roOut->mask->numRows, PS_TYPE_F32); + psImageInit(roOut->image, 1.0); + if (!ppStatsFPA(data->stats, data->out, view, + options->combine->maskVal | pmConfigMask("BLANK", config), + config)) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to generate stats for image.\n"); + return false; + } + psFree(roOut->image); + roOut->image = NULL; + } + + psFree(roOut); // Drop reference + + if (cellOut->hdu && !cellOut->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out cell HDU.\n"); + pmCellWriteMask(cellOut, data->outFile, config->database, false); + pmCellFreeData(cellOut); + } + } + + if (chipOut->hdu && !chipOut->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out chip HDU.\n"); + pmChipWriteMask(chipOut, data->outFile, config->database, false, false); + pmChipFreeData(chipOut); + } + } + + // Get list of FPAs for concepts averaging + { + psList *inFPAs = psListAlloc(NULL); // List of FPAs + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmFPA *fpaIn = data->in->data[i]; // Input FPA + psListAdd(inFPAs, PS_LIST_TAIL, fpaIn); + } + + if (!pmConceptsAverageFPAs(fpaOut, inFPAs)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts."); + psFree(inFPAs); + return false; + } + psFree(inFPAs); + } + + if (fpaOut->hdu && !fpaOut->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out FPA HDU.\n"); + pmFPAWriteMask(fpaOut, data->outFile, config->database, false, false); + } + pmFPAFreeData(fpaOut); + + psFree(view); + + return true; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskReadoutByPixelStats.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskReadoutByPixelStats.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskReadoutByPixelStats.c (revision 22293) @@ -0,0 +1,327 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include + +// XXX: Maybe add support for S16 and S32 types. Currently, only F32 supported. +bool ppMergeMaskReadoutByPixelStats(pmReadout *output, const psArray *inputs, const pmCombineParams *params) +{ + // Check inputs + PS_ASSERT_PTR_NON_NULL(output, false); + PS_ASSERT_ARRAY_NON_NULL(inputs, false); + PS_ASSERT_PTR_NON_NULL(params, false); + PS_ASSERT_FLOAT_WITHIN_RANGE(params->fracLow, 0.0, 1.0, false); + PS_ASSERT_FLOAT_WITHIN_RANGE(params->fracHigh, 0.0, 1.0, false); + if (params->combine != PS_STAT_SAMPLE_MEAN && params->combine != PS_STAT_SAMPLE_MEDIAN && + params->combine != PS_STAT_ROBUST_MEDIAN && params->combine != PS_STAT_FITTED_MEAN && + params->combine != PS_STAT_CLIPPED_MEAN) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Combination method is not SAMPLE_MEAN, SAMPLE_MEDIAN, " + "ROBUST_MEDIAN, FITTED_MEAN or CLIPPED_MEAN.\n"); + return false; + } + + // XXX is it possible / desired to use the weight in this analysis? + for (int i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + if (params->weights && !readout->weight) { + psError(PS_ERR_UNEXPECTED_NULL, true, + "Rejection based on weights requested, but no weights supplied for image %d.\n", i); + return false; + } + } + + bool first = !output->image; // First pass through? + + pmHDU *hdu = pmHDUFromReadout(output); // Output HDU + if (!hdu) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find HDU for readout.\n"); + return false; + } + + if (first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Combining using statistic: %x", params->combine); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + psStats *stats = psStatsAlloc(params->combine); // The statistics to use in the combination + if (params->combine == PS_STAT_CLIPPED_MEAN) { + stats->clipSigma = params->rej; + stats->clipIter = params->iter; + + if (first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Combination clipping: %d iterations, rejection at %f sigma", + params->iter, params->rej); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + } + + int minInputCols, maxInputCols, minInputRows, maxInputRows; // Smallest and largest values to combine + int xSize, ySize; // Size of the output image + if (!pmReadoutStackValidate(&minInputCols, &maxInputCols, &minInputRows, &maxInputRows, &xSize, &ySize, + inputs)) { + psError(PS_ERR_UNKNOWN, false, "No valid input readouts."); + return false; + } + + pmReadoutUpdateSize(output, minInputCols, minInputRows, xSize, ySize, true); + psTrace("psModules.imcombine", 7, "Output minimum: %d,%d\n", output->col0, output->row0); + + psStatsOptions combineStdev = 0; // Statistics option for weights + if (params->weights) { + + if (!output->weight) { + output->weight = psImageAlloc(xSize, ySize, PS_TYPE_F32); + } + if (output->weight->numCols < xSize || output->weight->numRows < ySize) { + psImage *newWeight = psImageAlloc(xSize, ySize, PS_TYPE_F32); + psImageInit(newWeight, 0.0); + psImageOverlaySection(newWeight, output->weight, output->col0, output->row0, "="); + psFree(output->weight); + output->weight = newWeight; + } + + if (first) { + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + "Using input weights to combine images", ""); + } + + // Get the correct statistics option for weights + switch (params->combine) { + case PS_STAT_SAMPLE_MEAN: + case PS_STAT_SAMPLE_MEDIAN: + combineStdev = PS_STAT_SAMPLE_STDEV; + break; + case PS_STAT_ROBUST_MEDIAN: + combineStdev = PS_STAT_ROBUST_STDEV; + break; + case PS_STAT_FITTED_MEAN: + combineStdev = PS_STAT_FITTED_STDEV; + break; + case PS_STAT_CLIPPED_MEAN: + combineStdev = PS_STAT_CLIPPED_STDEV; + break; + default: + psAbort("Should never get here --- checked params->combine before.\n"); + } + stats->options |= combineStdev; + } + + // We loop through each pixel in the output image. We loop through each input readout. We determine if + // that output pixel is contained in the image from that readout. If so, we save it in psVector pixels. + // If not, we set a mask for that element in pixels. Then, we mask off pixels not between fracLow and + // fracHigh. Then we call the vector stats routine on those pixels/mask. Then we set the output pixel + // value to the result of the stats call. + + psVector *pixels = psVectorAlloc(inputs->n, PS_TYPE_F32); // Stack of pixels + psF32 *pixelsData = pixels->data.F32; // Dereference pixels + + psVector *mask = psVectorAlloc(inputs->n, PS_TYPE_U8); // Mask for stack + psU8 *maskData = mask->data.U8; // Dereference mask + + psVector *weights = NULL; // Stack of weights + psVector *errors = NULL; // Stack of errors (sqrt of variance/weights), for psVectorStats + psF32 *weightsData = NULL; // Dereference weights + if (params->weights) { + weights = psVectorAlloc(inputs->n, PS_TYPE_F32); // Stack of weights + weightsData = weights->data.F32; + } + psVector *index = NULL; // The indices to sort the pixels + + float keepFrac = 1.0 - params->fracLow - params->fracHigh; // Fraction of pixels to keep + if (keepFrac != 1.0 && first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Min/max rejection: %f high, %f low, keep %d", + params->fracHigh, params->fracLow, params->nKeep); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + psMaskType maskVal = params->maskVal; // The mask value + if (maskVal && first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Mask for combination: %x", maskVal); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + #ifndef PS_NO_TRACE + psTrace("psModules.imcombine", 3, "Iterating output: %d --> %d, %d --> %d\n", + minInputCols - output->col0, maxInputCols - output->col0, + minInputRows - output->row0, maxInputRows - output->row0); + if (psTraceGetLevel("psModules.imcombine") >= 3) { + for (int r = 0; r < inputs->n; r++) { + pmReadout *readout = inputs->data[r]; // Input readout + psTrace("psModules.imcombine", 3, "Iterating input %d: %d --> %d, %d --> %d\n", r, + minInputCols - readout->col0, maxInputCols - readout->col0, + minInputRows - readout->row0, maxInputRows - readout->row0); + } + } + #endif + + // Dereference output products + psF32 **outputImage = output->image->data.F32; // Output image + psU8 **outputMask = output->mask->data.U8; // Output mask + psF32 **outputWeight = NULL; // Output weight map + if (output->weight) { + outputWeight = output->weight->data.F32; + } + + psVector *invScale = NULL; // Inverse scale; pre-calculated for efficiency + if (scale) { + invScale = (psVector*)psBinaryOp(NULL, psScalarAlloc(1.0, PS_TYPE_F32), "/", (const psPtr)scale); + } + + for (int i = minInputRows; i < maxInputRows; i++) { + int yOut = i - output->row0; // y position on output readout + #ifdef SHOW_BUSY + + if (psTraceGetLevel("psModules.imcombine") > 9) { + printf("Processing row %d\r", i); + fflush(stdout); + } + #endif + for (int j = minInputCols; j < maxInputCols; j++) { + int xOut = j - output->col0; // x position on output readout + + int numValid = 0; // Number of valid pixels in the stack + memset(maskData, 0, mask->n * sizeof(psU8)); // Reset the mask + for (int r = 0; r < inputs->n; r++) { + pmReadout *readout = inputs->data[r]; // Input readout + int yIn = i - readout->row0; // y position on input readout + int xIn = j - readout->col0; // x position on input readout + psImage *image = readout->image; // The readout image + + #if 0 // This should have been taken care of already: + // Check bounds + if (xIn < 0 || xIn >= image->numCols || yIn < 0 || yIn >= image->numRows) { + continue; + } + #endif + + pixelsData[r] = image->data.F32[yIn][xIn]; + if (!isfinite(pixelsData[r])) { + maskData[r] = 1; + continue; + } + + // Check mask + psImage *roMask = readout->mask; // The mask image + if (roMask && roMask->data.U8[yIn][xIn] & maskVal) { + maskData[r] = 1; + continue; + } + + if (params->weights) { + weightsData[r] = readout->weight->data.F32[yIn][xIn]; + } + + if (zero) { + pixelsData[r] -= zero->data.F32[r]; + } + if (scale) { + pixelsData[r] *= invScale->data.F32[r]; + if (params->weights) { + weightsData[r] *= invScale->data.F32[r] * invScale->data.F32[r]; + } + } + + numValid++; + } + + if (numValid == 0) { + outputMask[yOut][xOut] = params->blank; + outputImage[yOut][xOut] = NAN; + continue; + } + + // Apply fracLow,fracHigh if there are enough pixels + if (numValid * keepFrac >= params->nKeep && keepFrac != 1.0) { + index = psVectorSortIndex(index, pixels); + int numLow = numValid * params->fracLow; // Number of low pixels to clip + int numHigh = numValid * params->fracHigh; // Number of high pixels to clip + // Low pixels + psS32 *indexData = index->data.S32; // Dereference index + for (int k = 0, numMasked = 0; numMasked < numLow && k < index->n; k++) { + // Don't count the ones that are already masked + if (!maskData[indexData[k]]) { + maskData[indexData[k]] = 1; + numMasked++; + } + } + // High pixels + for (int k = pixels->n - 1, numMasked = 0; numMasked < numHigh && k >= 0; k--) { + // Don't count the ones that are already masked + if (! maskData[indexData[k]]) { + maskData[indexData[k]] = 1; + numMasked++; + } + } + } + + // XXXXX this step probably is very expensive : convert errors to variance everywhere? + if (params->weights) { + errors = (psVector*)psUnaryOp(errors, weights, "sqrt"); + } + + // Combination + if (!psVectorStats(stats, pixels, errors, mask, 1)) { + // Can't do much about it, but it's not worth worrying about + psErrorClear(); + outputImage[yOut][xOut] = NAN; + outputMask[yOut][xOut] = params->blank; + if (params->weights) { + outputWeight[yOut][xOut] = NAN; + } + } else { + outputImage[yOut][xOut] = psStatsGetValue(stats, params->combine); + if (!isfinite(outputImage[yOut][xOut])) { + outputMask[yOut][xOut] = params->blank; + } + if (params->weights) { + float stdev = psStatsGetValue(stats, combineStdev); + outputWeight[yOut][xOut] = PS_SQR(stdev); // Variance + // XXXX this is not the correct formal error. + // also, the weighted mean is not obviously the correct thing here + } + } + } + } + #ifdef SHOW_BUSY + if (psTraceGetLevel("psModules.imcombine") > 9) { + printf("\n"); + } + #endif + psFree(index); + psFree(pixels); + psFree(mask); + psFree(weights); + psFree(errors); + psFree(stats); + psFree(invScale); + + // Update the "concepts" + psList *inputCells = psListAlloc(NULL); // List of cells + for (long i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + psListAdd(inputCells, PS_LIST_TAIL, readout->parent); + } + bool success = pmConceptsAverageCells(output->parent, inputCells, NULL, NULL, true); + psFree(inputCells); + + output->data_exists = true; + output->parent->data_exists = true; + output->parent->parent->data_exists = true; + + return success; +} + Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskSuspect.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskSuspect.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskSuspect.c (revision 22293) @@ -0,0 +1,117 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask +bool ppMergeMaskSuspect(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator + pmFPA *fpaOut = data->out; // Output FPA + + // Iterate over each file + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + psFits *fits = data->files->data[i]; // FITS file handle + if (!fits) { + continue; + } + psTrace("ppMerge", 3, "File %d: %s\n", i, (const char*)filenames->data[i]); + + pmFPA *fpaIn = data->in->data[i]; // Input FPA + pmFPAview *view = pmFPAviewAlloc(0); // View of FPA, for iteration + pmChip *chipIn; // Input chip of interest + while ((chipIn = pmFPAviewNextChip(view, fpaIn, 1))) { + + // handle chip vs cell statistics & avoid reading the data twice + + // load the data of all cells + pmCell *cellIn; // Input cell of interest + while ((cellIn = pmFPAviewNextCell(view, fpaIn, 1))) { + if (!pmCellRead(cellIn, fits, config->database)) continue; + if (cellIn->readouts->n == 0) continue; + } + + // calculate the readout statistics either for each readout, or across the entire chip + if (options->statsByChip) { + pmChip *chipOut = pmFPAviewThisChip(view, fpaOut); // Output cell + if (!ppMergeMaskChipStats (chipIn, chipOut, options, rng)) { + psAbort ("stats problem"); + } + } else { + // calculate the stats for each cell independently + while ((cellIn = pmFPAviewNextCell(view, fpaIn, 1))) { + if (cellIn->readouts->n == 0) continue; + pmCell *cellOut = pmFPAviewThisCell(view, fpaOut); // Output cell + + pmReadout *roOut = NULL; + if (cellOut->readouts->n > 0) { + roOut = cellOut->readouts->data[0]; + psTrace("ppMerge", 5, "masking with results from previous pass\n"); + } + + pmReadout *roIn; // Input readout of interest + while ((roIn = pmFPAviewNextReadout(view, fpaIn, 1))) { + if (!roIn->image) continue; + psTrace("ppMerge", 4, "Measure statistics for chip %d, cell %d, ro %d\n", + view->chip, view->cell, view->readout); + ppMergeMaskReadoutStats (roIn, roOut, options, rng); + } + } + } + + // apply the measured statistics to determine the outliers to be masked + while ((cellIn = pmFPAviewNextCell(view, fpaIn, 1))) { + if (cellIn->readouts->n == 0) continue; + + pmCell *cellOut = pmFPAviewThisCell(view, fpaOut); // Output cell + // Suspect pixels image + psImage *suspect = psMetadataLookupPtr(NULL, cellOut->analysis, "MASK.SUSPECT"); + bool first = suspect ? false : true; + + pmReadout *roIn; // Input readout of interest + while ((roIn = pmFPAviewNextReadout(view, fpaIn, 1))) { + if (!roIn->image) { + continue; + } + psTrace("ppMerge", 4, "Flagging suspect pixels in chip %d, cell %d, ro %d\n", + view->chip, view->cell, view->readout); + suspect = pmMaskFlagSuspectPixels(suspect, roIn, options->maskSuspect, options->combine->maskVal); + } + + if (first) { + psMetadataAddImage(cellOut->analysis, PS_LIST_TAIL, "MASK.SUSPECT", 0, + "Suspect pixels", suspect); + psFree(suspect); + } + pmCellFreeData(cellIn); + } + pmChipFreeData(chipIn); + } + pmFPAFreeData(fpaIn); + psFree(view); + } + psFree(rng); + + return true; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskWrite.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskWrite.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeMaskWrite.c (revision 22293) @@ -0,0 +1,69 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask +bool ppMergeMaskWrite(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + pmFPA *fpaOut = data->out; // Output FPA + + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + if (fpaOut->hdu) { + pmFPAUpdateNames(fpaOut, NULL, NULL); + } + pmFPAWriteMask(fpaOut, data->outFile, config->database, true, false); // Write header only + pmChip *chipOut; // Output chip of interest + while ((chipOut = pmFPAviewNextChip(view, fpaOut, 1))) { + if (chipOut->hdu) { + chipOut->data_exists = true; + pmFPAUpdateNames(fpaOut, chipOut, NULL); + } + pmChipWriteMask(chipOut, data->outFile, config->database, true, false); // Write header only + pmCell *cellOut; // Output cell of interest + while ((cellOut = pmFPAviewNextCell(view, fpaOut, 1))) { + if (cellOut->hdu) { + chipOut->data_exists = cellOut->data_exists = true; + pmFPAUpdateNames(fpaOut, chipOut, cellOut); + } + pmCellWriteMask(cellOut, data->outFile, config->database, true); // Write header only + + if (cellOut->hdu && !cellOut->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out cell HDU.\n"); + pmCellWriteMask(cellOut, data->outFile, config->database, false); + pmCellFreeData(cellOut); + } + } + + if (chipOut->hdu && !chipOut->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out chip HDU.\n"); + pmChipWriteMask(chipOut, data->outFile, config->database, false, false); + pmChipFreeData(chipOut); + } + } + + if (fpaOut->hdu && !fpaOut->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out FPA HDU.\n"); + pmFPAWriteMask(fpaOut, data->outFile, config->database, false, false); + } + pmFPAFreeData(fpaOut); + + psFree(view); + + return true; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeOptions.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeOptions.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeOptions.c (revision 22293) @@ -0,0 +1,323 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeOptions.h" + +#define ARRAY_BUFFER 4 // Number of values to add at once + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ppMergeOptions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Free function +static void mergeOptionsFree(ppMergeOptions *options // Options to free + ) +{ + psFree(options->format); + psFree(options->combine); + psFree(options->darkOrdinates); + psFree(options->darkNorm); +} + +// Allocator +ppMergeOptions *ppMergeOptionsAlloc(const pmConfig *config) +{ + ppMergeOptions *options = psAlloc(sizeof(ppMergeOptions)); // The options, to return + psMemSetDeallocator(options, (psFreeFunc)mergeOptionsFree); + + options->format = NULL; + options->rows = 0; + options->minElectrons = NAN; + options->zero = false; + options->scale = false; + options->dark = false; + options->fringe = false; + options->shutter = false; + options->mask = false; + options->sample = 1; + options->mean = PS_STAT_SAMPLE_MEDIAN; + options->stdev = PS_STAT_SAMPLE_STDEV; + options->fringeNum = 100; + options->fringeSize = 10; + options->fringeSmoothX = 5; + options->fringeSmoothY = 5; + options->shutterSize = 10; + options->shutterIter = 2; + options->shutterRej = 3.0; + options->maskSuspect = 5.0; + options->maskBad = 10.0; + options->maskMode = PM_MASK_ID_VALUE; + options->statsByChip = true; + options->onOff = 0; + options->combine = pmCombineParamsAlloc(PS_STAT_SAMPLE_MEAN); + options->combine->rej = 3.0; + options->combine->iter = 1; + options->combine->fracHigh = 0.0; + options->combine->fracLow = 0.0; + options->combine->nKeep = 1; + options->combine->maskVal = 0xff; + options->combine->weights = false; + options->combine->blank = pmConfigMask("BLANK", config); + options->satMask = 0x00; + options->badMask = 0x00; + options->growPixels = 0; + options->darkOrdinates = NULL; + options->darkNorm = NULL; + return options; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ppMergeOptionsParse +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Parse a recipe option according to its type +#define OPTION_PARSE(OPTION,MD,NAME,TYPE) \ +{ \ + psMetadataItem *item = psMetadataLookup(MD, NAME); \ + if (item) { \ + OPTION = psMetadataItemParse##TYPE(item); \ + } else { \ + psWarning("Recipe option %s isn't specified; using default.\n", NAME); \ + } \ +} + +// Parse a statistic +static psStatsOptions parseStat(psMetadata *source, // Source of the statistics option + const char *name // Name of the statistics option + ) +{ + bool mdok = true; // Status of MD lookup + const char *stat = psMetadataLookupStr(&mdok, source, name); // The statistic string + if (!mdok || !stat || strlen(stat) == 0) { + return 0; + } + return psStatsOptionFromString(stat); +} + +// Parse the options +ppMergeOptions *ppMergeOptionsParse(pmConfig *config // Configuration + ) +{ + ppMergeOptions *options = ppMergeOptionsAlloc(config); // The merge options + + // We need to work out the camera before we can get the recipe. Take the first input and inspect it. + if (!config->camera) { + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + psString resolved = pmConfigConvertFilename(filenames->data[0], config, false); // Resolved file name + psFits *inFile = psFitsOpen(resolved, "r"); // The FITS file to read + if (!inFile) { + psError(PS_ERR_IO, false, "Unable to open input file %s to determine camera.\n", resolved); + psFree(resolved); + psFree(config); + exit(EXIT_FAILURE); + } + psFree(resolved); + psMetadata *header = psFitsReadHeader(NULL, inFile); // The FITS (primary) header + psFitsClose(inFile); + + options->format = pmConfigCameraFormatFromHeader(config, header, true); + psFree(header); + if (!options->format) { + psLogMsg(__func__, PS_LOG_WARN, "Unable to identify camera format for input file %s\n", + (char *)filenames->data[0]); + exit(EXIT_FAILURE); + } + } + + // we must have the recipes by this point + assert (config->recipes); + + // Now we can read the recipe + bool mdok = true; // Status of MD lookup + psMetadata *recipe = psMetadataLookupMetadata(&mdok, config->recipes, PPMERGE_RECIPE); // Recipe information + if (!mdok || !recipe) { + psError(PS_ERR_IO, true, "Unable to find recipe %s", PPMERGE_RECIPE); + exit(EXIT_FAILURE); + } + + // First, deal with the recipe. These are parameters that will typically be constant for a camera. + OPTION_PARSE(options->rows, recipe, "ROWS", S32); + OPTION_PARSE(options->minElectrons, recipe, "ELECTRONS", F32); + OPTION_PARSE(options->sample, recipe, "SAMPLE", S32); + OPTION_PARSE(options->combine->rej, recipe, "REJ", F32); + OPTION_PARSE(options->combine->iter, recipe, "ITER", S32); + OPTION_PARSE(options->combine->fracHigh, recipe, "FRACHIGH", F32); + OPTION_PARSE(options->combine->fracLow, recipe, "FRACLOW", F32); + OPTION_PARSE(options->combine->nKeep, recipe, "NKEEP", S32); + OPTION_PARSE(options->combine->weights, recipe, "WEIGHTS", Bool); + OPTION_PARSE(options->fringeNum, recipe, "FRINGE.NUM", S32); + OPTION_PARSE(options->fringeSize, recipe, "FRINGE.SIZE", S32); + OPTION_PARSE(options->fringeSmoothX, recipe, "FRINGE.XSMOOTH", S32); + OPTION_PARSE(options->fringeSmoothY, recipe, "FRINGE.YSMOOTH", S32); + OPTION_PARSE(options->shutterSize, recipe, "SHUTTER.SIZE", S32); + OPTION_PARSE(options->shutterIter, recipe, "SHUTTER.ITER", S32); + OPTION_PARSE(options->shutterRej, recipe, "SHUTTER.REJECT", F32); + OPTION_PARSE(options->maskSuspect, recipe, "MASK.SUSPECT", F32); + OPTION_PARSE(options->maskBad, recipe, "MASK.BAD", F32); + OPTION_PARSE(options->statsByChip, recipe, "STATS.BY.CHIP", Bool); + OPTION_PARSE(options->growPixels, recipe, "MASK.GROW.NPIX", S32); + + const char *masks = psMetadataLookupStr(NULL, recipe, "MASKVAL"); + options->combine->maskVal = pmConfigMask(masks, config); + + options->combine->combine = parseStat(recipe, "COMBINE"); + options->mean = parseStat(recipe, "MEAN"); + options->stdev = parseStat(recipe, "STDEV"); + + // Now the command-line options. These are parameters that depend on what type of frame is being combined + + // Set options based on the type of calibration frame + const char *type = psMetadataLookupStr(NULL, config->arguments, "-type"); // The type of calibration frame + if (strlen(type) > 0) { + if (strcasecmp(type, "BIAS") == 0) { + options->zero = false; + options->scale = false; + options->dark = false; + options->fringe = false; + options->shutter = false; + options->mask = false; + } else if (strcasecmp(type, "DARK") == 0) { + options->zero = false; + options->scale = false; + options->dark = true; + options->fringe = false; + options->shutter = false; + options->mask = false; + } else if (strcasecmp(type, "FLAT") == 0) { + options->zero = false; + options->scale = true; + options->dark = false; + options->fringe = false; + options->shutter = false; + options->mask = false; + } else if (strcasecmp(type, "SKYFLAT") == 0) { + options->zero = false; + options->scale = true; + options->dark = false; + options->fringe = false; + options->shutter = false; + options->mask = false; + } else if (strcasecmp(type, "DOMEFLAT") == 0) { + options->zero = false; + options->scale = true; + options->dark = false; + options->fringe = false; + options->shutter = false; + options->mask = false; + } else if (strcasecmp(type, "FRINGE") == 0) { + options->zero = true; + options->scale = true; + options->dark = false; + options->fringe = true; + options->shutter = false; + options->mask = false; + } else if (strcasecmp(type, "SHUTTER") == 0) { + options->zero = false; + options->scale = false; + options->dark = false; + options->fringe = false; + options->shutter = true; + options->mask = false; + } else if (strcasecmp(type, "MASK") == 0 || + strcasecmp(type, "DARKMASK") == 0 || + strcasecmp(type, "FLATMASK") == 0) { + options->zero = false; + options->scale = false; + options->dark = false; + options->fringe = false; + options->shutter = false; + options->mask = true; + } else { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unrecognised image type: %s", type); + psFree(options); + return NULL; + } + } else { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "No image type specified."); + psFree(options); + return NULL; + } + + const char *maskMode = psMetadataLookupStr(NULL, recipe, "MASK.MODE"); // The type of calibration frame + if (!maskMode) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "No mask mode specified in recipe."); + psFree(options); + return NULL; + } + + options->maskMode = pmMaskIdentifyModeFromString (maskMode); + if (options->maskMode == PM_MASK_ID_NONE) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "invalid mask mode %s.", maskMode); + psFree(options); + return NULL; + } + +#if 0 + // Number of on/off images + OPTION_PARSE(options->onOff, config->arguments, "-onoff", S32); +#endif + + // Masking options + options->satMask = pmConfigMask("SAT", config); + options->badMask = pmConfigMask("BAD", config); + + if (options->dark) { + psMetadata *ordinates = psMetadataLookupMetadata(NULL, recipe, "DARK.ORDINATES"); // Ordinates info + options->darkOrdinates = psArrayAllocEmpty(psListLength(ordinates->list)); + + psMetadataIterator *iter = psMetadataIteratorAlloc(ordinates, PS_LIST_HEAD, NULL); // Iterator + psMetadataItem *item; // Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + int order = 0; // Polynomial order + bool scale = false; // Scale values? + float min = NAN, max = NAN; // Minimum and maximum values for scaling + switch (item->type) { + case PS_TYPE_S32: + order = item->data.S32; + break; + case PS_DATA_METADATA: + order = psMetadataLookupS32(NULL, item->data.md, "ORDER"); + bool mdok; // Status of MD lookup + scale = psMetadataLookupBool(&mdok, item->data.md, "SCALE"); + min = psMetadataLookupF32(&mdok, item->data.md, "MIN"); + max = psMetadataLookupF32(&mdok, item->data.md, "MAX"); + break; + default: + psError(PS_ERR_BAD_PARAMETER_TYPE, true, + "Type of DARK.ORDINATES entry %s (%x) is not METADATA or S32", + item->name, item->type); + psFree(options); + return false; + } + if (order <= 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "ORDER not positive (%d) for DARK.ORDINATES %s", + order, item->name); + psFree(options); + return false; + } + + pmDarkOrdinate *ord = pmDarkOrdinateAlloc(item->name, order); + ord->scale = scale; + ord->min = min; + ord->max = max; + psArrayAdd(options->darkOrdinates, options->darkOrdinates->n, ord); + psFree(ord); + } + psFree(iter); + + psString darkNorm = psMetadataLookupStr(NULL, recipe, "DARK.NORM"); // Normalisation concept + if (darkNorm && strcmp(darkNorm, "NONE") != 0) { + options->darkNorm = psStringCopy(darkNorm); + } + } + + return options; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeOptions.h =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeOptions.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeOptions.h (revision 22293) @@ -0,0 +1,59 @@ +#ifndef PP_MERGE_OPTIONS_H +#define PP_MERGE_OPTIONS_H + +#include +#include + +// Mode of on/off pairs; the value corresponds to how many images are in each set +typedef enum { + PP_ONOFF_ABBA = -1, // On/off pairs in the ABBA mode + PP_ONOFF_NONE = 0, // No on/off pairs + PP_ONOFF_ABAB = 1, // On/off pairs in the ABAB mode (one image each) + PP_ONOFF_AABB = 2, // On/off pairs, two images each + // And so on and so forth... just use a number beyond this +} ppOnOff; + +// Options for ppMerge +typedef struct { + psMetadata *format; // Camera format configuration + unsigned int rows; // Number of rows to read at once + float minElectrons; // Minimum number of electrons for useful signal + bool zero; // Subtract background before combining? + bool scale; // Scale by the background before combining? + bool dark; // Generate dark? + bool fringe; // Make fringe measurements? + bool shutter; // Generate shutter correction? + bool mask; // Generate bad pixel mask? + unsigned int sample; // Sampling factor for measuring the background + psStatsOptions mean; // Statistic to use to measure the mean + psStatsOptions stdev; // Statistic to use to measure the stdev + int fringeNum; // Number of fringe regions per cell + int fringeSize; // Size of fringe regions + int fringeSmoothX; // Number of smoothing regions per cell, in x + int fringeSmoothY; // Number of smoothing regions per cell, in y + int shutterSize; // Size for shutter measurement regions + int shutterIter; // Number of iterations for shutter measurement + float shutterRej; // Rejection limit for shutter measurement + float maskSuspect; // Threshold for identifying suspect pixels + float maskBad; // Threshold for identifying bad pixels + bool statsByChip; // measure statistics for masking by chip or readout? + pmMaskIdentifyMode maskMode; // how to set the limit based on the threshold value above? + ppOnOff onOff; // On/off pairs? + pmCombineParams *combine; // Combination parameters + psMaskType satMask; // Mask value for saturated pixels + psMaskType badMask; // Mask value for bad (low) pixels + int growPixels; + psArray *darkOrdinates; // Ordinates for dark combination + psString darkNorm; // Dark normalisation concept +} ppMergeOptions; + +// Allocator +ppMergeOptions *ppMergeOptionsAlloc(const pmConfig *config); + + +// Parse the options for ppMerge +ppMergeOptions *ppMergeOptionsParse(pmConfig *config // Configuration + ); + + +#endif Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeScaleZero.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeScaleZero.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeScaleZero.c (revision 22293) @@ -0,0 +1,348 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeScaleZero.h" + +// Get the scale and zero for each chip of each input +bool ppMergeScaleZero(psImage **scales, // The scales for each integration/cell + psImage **zeros, // The zeroes for each integration/cell + psArray **shutters, // The shutter correction data for each cell + ppMergeData *data,// The data + const ppMergeOptions *options, // The options + const pmConfig *config // The configuration + ) +{ + assert(data); + assert(options); + assert(config); + + if (!options->scale && !options->zero && !options->shutter) { + return true; // We did everything we were asked for + } + + assert(config->camera); // Need the camera configuration + assert(config->arguments); // Need the list of files + + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + // Sanity checks + assert(!options->scale || scales); + assert(!scales || !*scales || ((*scales)->type.type == PS_TYPE_F32 && + (*scales)->numCols == data->numCells && + (*scales)->numRows == filenames->n)); + assert(!options->zero || zeros); + assert(!zeros || !*zeros || ((*zeros)->type.type == PS_TYPE_F32 && + (*zeros)->numCols == data->numCells && + (*zeros)->numRows == filenames->n)); + assert(!options->shutter || shutters); + assert(!shutters || !*shutters || (*shutters)->n == data->numCells); + + // Allocate the outputs + if (options->scale) { + if (*scales) { + psFree(*scales); + } + *scales = psImageAlloc(data->numCells, filenames->n, PS_TYPE_F32); + } + if (options->zero) { + if (*zeros) { + psFree(*zeros); + } + *zeros = psImageAlloc(data->numCells, filenames->n, PS_TYPE_F32); + } + psRandom *rng = NULL; // Random number generator + if (options->shutter) { + if (*shutters) { + psFree(*shutters); + } + *shutters = psArrayAlloc(data->numCells); + rng = psRandomAlloc(PS_RANDOM_TAUS, 0); + } + + bool fromConcepts = false; // Do we get the scale and zero points from the concepts? + bool first = true; // Are we on the first cell (that sets the standard for the rest)? + bool done = false; // Are we done going through the list? + bool mdok = true; // Status of MD lookup + for (long i = 0; i < data->in->n && !done; i++) { + pmFPA *fpa = data->in->data[i]; // The FPA + if (!fpa) { + continue; + } + long cellNum = -1; // Number of the cell + psArray *chips = fpa->chips; // The array of chips + for (long j = 0; j < chips->n && !done; j++) { + pmChip *chip = chips->data[j]; // The chip + if (!chip) { + continue; + } + psArray *cells = chip->cells; // The array of cells + for (long k = 0; k < cells->n && !done; k++) { + pmCell *cell = cells->data[k]; // The cell + if (!cell) { + continue; + } + cellNum++; + + if (options->scale) { + float scale = psMetadataLookupF32(&mdok, cell->concepts, "PPMERGE.SCALE"); // The scale + if (mdok && !isnan(scale)) { + if (!first && !fromConcepts) { + psWarning("PPMERGE.SCALE and PPMERGE.ZERO have been set " + "for some, but not all cells --- we will re-measure it for all cells."); + done = true; + continue; + } + fromConcepts = true; + (*scales)->data.F32[i][cellNum] = scale; + psTrace("ppMerge", 9, "Scale for input %ld, chip %ld, cell %ld: %f\n", + i, j, k, scale); + } else if (!first && fromConcepts) { + psWarning("PPMERGE.SCALE and PPMERGE.ZERO have been set " + "for some, but not all cells --- we will re-measure it for all cells."); + fromConcepts = false; + done = true; + continue; + } + } + + if (options->zero) { + float zero = psMetadataLookupF32(&mdok, cell->concepts, "PPMERGE.ZERO"); // The zero + if (mdok && !isnan(zero)) { + if (!first && !fromConcepts) { + psWarning("PPMERGE.SCALE and PPMERGE.ZERO have been set " + "for some, but not all cells --- we will re-measure it for all cells."); + done = true; + continue; + } + fromConcepts = true; + (*zeros)->data.F32[i][cellNum] = zero; + psTrace("ppMerge", 9, "Zero for input %ld, chip %ld, cell %ld: %f\n", i, j, k, zero); + } else if (!first && fromConcepts) { + psWarning("PPMERGE.SCALE and PPMERGE.ZERO have been set " + "for some, but not all cells --- we will re-measure it for all cells."); + fromConcepts = false; + done = true; + continue; + } + } + + first = false; + } + } + } + + if (fromConcepts) { + // We've already done everything we need to + psFree(rng); + return true; + } + + psImage *background = psImageAlloc(data->numCells, filenames->n, PS_TYPE_F32); // Background measurements + psImageInit(background, NAN); + psStats *bgStats = psStatsAlloc(options->mean); // Statistic to measure the background + psVector *gains = NULL; // The gains for each cell + if (options->scale) { + gains = psVectorAlloc(data->numCells, PS_TYPE_F32); + } + + pmFPAview *view = pmFPAviewAlloc(0); + + bool status = true; // Status of getting the scale and zero --- did everything go right? + for (int i = 0; i < filenames->n; i++) { + psString name = filenames->data[i]; // The name of the file + if (!name || strlen(name) == 0) { + continue; + } + psTrace("ppMerge", 9, "Opening %s to get background...\n", name); + psFits *inFile = data->files->data[i]; // The FITS file to read + pmFPA *fpa = data->in->data[i]; // The FPA for this input + int cellNum = -1; // Number of the cell + psArray *chips = fpa->chips; // Array of chips + for (int j = 0; j < chips->n; j++) { + pmChip *chip = chips->data[j]; // The chip of interest + if (!chip) { + continue; + } + psArray *cells = chip->cells; // Array of cells + for (int k = 0; k < cells->n; k++) { + pmCell *cell = cells->data[k]; // The cell of interest + if (!cell) { + continue; + } + cellNum++; + + if (!pmCellReadHeader(cell, inFile)) { + continue; + } + + // Scaling by the background + if (options->scale || options->zero) { + if (!pmCellRead(cell, inFile, config->database)) { + // Nothing here + pmCellFreeData(cell); + continue; + } + + if (cell->readouts->n > 1) { + psWarning("File %s chip %d cell %d contains more than one " + "readout --- ignoring all but the first.\n", name, j, k); + status = false; + } + + pmReadout *readout = cell->readouts->data[0]; // The readout of interest + psImage *image = readout->image; // The pixels of interest + if (!image) { + pmCellFreeData(cell); + continue; + } + + // Get the gain + if (options->scale) { + bool mdok = true; // Status of MD lookup + gains->data.F32[cellNum] = psMetadataLookupF32(&mdok, cell->concepts, "CELL.GAIN"); + if (!mdok || isnan(gains->data.F32[cellNum])) { + psWarning("CELL.GAIN for file %s chip %d cell %d is not " + "set.\n", name, j, k); + gains->data.F32[cellNum] = NAN; + status = false; + } + } + + // Get the background + int sampleSize = (image->numCols * image->numRows) / options->sample; // Size of sample + psVector *sample = psVectorAlloc(sampleSize, PS_TYPE_F32); // Sample of the image + psVector *sampleMask = NULL; // Mask for sample + if (readout->mask) { + sampleMask = psVectorAlloc(sampleSize, PS_TYPE_U8); + } + psImage *mask = readout->mask; // The mask image + for (long i = 0; i < sampleSize; i++) { + int j = i * options->sample; // Index into image + int x = j % image->numCols; // x index + int y = j / image->numCols; // y index + sample->data.F32[i] = image->data.F32[y][x]; + if (readout->mask) { + sampleMask->data.U8[i] = mask->data.U8[y][x]; + } + } + status = psVectorStats(bgStats, sample, sampleMask, NULL, options->combine->maskVal); + if (!status) { + psTrace("ppMerge", 3, "failed to get stats for for %s, cell %d is %f\n", name, cellNum, + background->data.F32[i][cellNum]); + psErrorClear(); + } + psFree(sample); + psFree(sampleMask); + background->data.F32[i][cellNum] = psStatsGetValue(bgStats, options->mean); + psTrace("ppMerge", 3, "Background for %s, cell %d is %f\n", name, cellNum, + background->data.F32[i][cellNum]); + } + + // Shutter correction + if (options->shutter) { + if (!pmCellRead(cell, inFile, config->database)) { + // Nothing here + pmCellFreeData(cell); + continue; + } + + if (cell->readouts->n > 1) { + psWarning("File %s chip %d cell %d contains more than one " + "readout --- ignoring all but the first.\n", name, j, k); + status = false; + } + pmReadout *readout = cell->readouts->data[0]; // The readout of interest + + pmShutterCorrectionData *shutter = (*shutters)->data[cellNum]; // Shutter correction data + if (!shutter) { + shutter = pmShutterCorrectionDataAlloc(readout->image->numCols, + readout->image->numRows, + options->shutterSize); + (*shutters)->data[cellNum] = shutter; + } + if (!pmShutterCorrectionAddReadout(shutter, readout, options->mean, options->stdev, + options->combine->maskVal, rng)) { + psWarning("Can't add file %s chip %d cell %d to shutter correction --- ignored.", + name, j, k); + status = false; + } + } + + + pmCellFreeData(cell); + } + pmChipFreeData(chip); + } + pmFPAFreeData(fpa); + } + psFree(rng); + psFree(bgStats); + psFree(view); + + if (options->scale) { + // Need to normalize over the focal plane + if (psTraceGetLevel("ppMerge") > 9) { + for (int i = 0; i < gains->n; i++) { + psTrace("ppMerge", 10, "Gain for cell %d is %f\n", i, gains->data.F32[i]); + } + } + psVector *fluxes = NULL; // Solution to fluxes + if (!pmFlatNormalize(&fluxes, &gains, background)) { + psWarning("Normalisation failed to converge --- continuing anyway.\n"); + status = false; + } + psFree(gains); + + psImage *scalesDeref = *scales; // Dereference the pointer + + for (int i = 0; i < scalesDeref->numRows; i++) { + psF32 bg = fluxes->data.F32[i]; + for (int j = 0; j < scalesDeref->numCols; j++) { + scalesDeref->data.F32[i][j] = bg; + } + } + } + + if (options->zero) { + if (!*zeros) { + // This is much faster than copying! + *zeros = psMemIncrRefCounter(background); + } else { + *zeros = psImageCopy(*zeros, background, PS_TYPE_F32); + } + } + + // Diagnostic stuff + if (scales && *scales && psTraceGetLevel("ppMerge") > 9) { + psImage *scalesDeref = *scales; // Dereference the pointer + for (int i = 0; i < scalesDeref->numRows; i++) { + for (int j = 0; j < scalesDeref->numCols; j++) { + psTrace("ppMerge", 9, "Scale for exposure %d, cell %d is: %f\n", i, j, + scalesDeref->data.F32[i][j]); + } + } + } + if (zeros && *zeros && psTraceGetLevel("ppMerge") > 9) { + psImage *zerosDeref = *zeros; // Dereference the pointer + for (int i = 0; i < zerosDeref->numRows; i++) { + for (int j = 0; j < zerosDeref->numCols; j++) { + psTrace("ppMerge", 9, "Zero for exposure %d, cell %d is: %f\n", i, j, + zerosDeref->data.F32[i][j]); + } + } + } + + psFree(background); + + return status; +} + Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeScaleZero.h =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeScaleZero.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeScaleZero.h (revision 22293) @@ -0,0 +1,19 @@ +#ifndef PP_MERGE_SCALE_ZERO_H +#define PP_MERGE_SCALE_ZERO_H + +#include +#include + +#include "ppMergeData.h" +#include "ppMergeOptions.h" + +// Get the scale and zero for each chip of each input +bool ppMergeScaleZero(psImage **scales, // The scales for each integration/cell + psImage **zeros, // The zeroes for each integration/cell + psArray **shutter,// The shutter correction data for each cell + ppMergeData *data,// The data + const ppMergeOptions *options, // The options + const pmConfig *config // The configuration + ); + +#endif Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeVersion.c =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeVersion.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeVersion.c (revision 22293) @@ -0,0 +1,60 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppMergeVersion.h" + +static const char *cvsTag = "$Name: not supported by cvs2svn $";// CVS tag name + +psString ppMergeVersion(void) +{ + psString version = NULL; // Version, to return + psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION); + return version; +} + +psString ppMergeVersionLong(void) +{ + psString version = ppMergeVersion(); // Version, to return + psString tag = psStringStripCVS(cvsTag, "Name"); // CVS tag + psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__); + psFree(tag); + return version; +} + + +void ppMergeVersionMetadata(psMetadata *metadata) +{ + PS_ASSERT_METADATA_NON_NULL(metadata,); + + psString pslib = psLibVersionLong();// psLib version + psString psmodules = psModulesVersionLong(); // psModules version + psString ppStats = ppStatsVersionLong(); // ppStats version + psString ppMerge = ppMergeVersionLong(); // ppMerge version + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psString head = NULL; // Head string + psStringAppend(&head, "ppMerge processing at %s. Component information:", timeString); + psFree(timeString); + + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, head, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, pslib, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, psmodules, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppStats, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppMerge, ""); + + psFree(head); + psFree(pslib); + psFree(psmodules); + psFree(ppStats); + psFree(ppMerge); + + return; +} Index: /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeVersion.h =================================================================== --- /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeVersion.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/ppMerge/src/ppMergeVersion.h (revision 22293) @@ -0,0 +1,14 @@ +#ifndef PP_MERGE_VERSION_H +#define PP_MERGE_VERSION_H + +/// Return short version information +psString ppMergeVersion(void); + +/// Return long version information +psString ppMergeVersionLong(void); + +/// Update the metadata with version information for all dependencies +void ppMergeVersionMetadata(psMetadata *metadata ///< Metadata to update with version information + ); + +#endif Index: /tags/pap_tags/pap_root_080320/psModules/src/camera/pmReadoutStack.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/camera/pmReadoutStack.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/camera/pmReadoutStack.c (revision 22293) @@ -0,0 +1,152 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pmReadoutStack.h" + + +bool pmReadoutUpdateSize(pmReadout *readout, int minCols, int minRows, + int numCols, int numRows, bool mask, bool weight, + psMaskType blank) +{ + PS_ASSERT_PTR_NON_NULL(readout, false); + + if (readout->image) { + readout->col0 = PS_MIN(minCols, readout->col0); + readout->row0 = PS_MIN(minRows, readout->row0); + } else { + readout->col0 = minCols; + readout->row0 = minRows; + } + + // (reAllocate the images + if (!readout->image) { + readout->image = psImageAlloc(numCols, numRows, PS_TYPE_F32); + psImageInit(readout->image, NAN); + } + if (readout->image->numCols < numCols || readout->image->numRows < numRows) { + // Generate the new output image by extending the current one, or making a whole new one + psImage *newImage = psImageAlloc(numCols, numRows, PS_TYPE_F32); + psImageInit(newImage, NAN); + psImageOverlaySection(newImage, readout->image, readout->col0, readout->row0, "="); + psFree(readout->image); + readout->image = newImage; + } + + if (mask) { + if (!readout->mask) { + readout->mask = psImageAlloc(numCols, numRows, PS_TYPE_MASK); + psImageInit(readout->mask, blank); + } + if (readout->mask->numCols < numCols || readout->mask->numRows < numRows) { + psImage *newMask = psImageAlloc(numCols, numRows, PS_TYPE_MASK); + psImageInit(newMask, blank); + psImageOverlaySection(newMask, readout->mask, readout->col0, readout->row0, "="); + psFree(readout->mask); + readout->mask = newMask; + } + } + + if (weight) { + if (!readout->weight) { + readout->weight = psImageAlloc(numCols, numRows, PS_TYPE_F32); + psImageInit(readout->weight, NAN); + } + if (readout->weight->numCols < numCols || readout->weight->numRows < numRows) { + psImage *newWeight = psImageAlloc(numCols, numRows, PS_TYPE_F32); + psImageInit(newWeight, NAN); + psImageOverlaySection(newWeight, readout->weight, readout->col0, readout->row0, "="); + psFree(readout->weight); + readout->weight = newWeight; + } + } + + return true; +} + +bool pmReadoutStackValidate(int *minInputColsPtr, int *maxInputColsPtr, int *minInputRowsPtr, + int *maxInputRowsPtr, int *numColsPtr, int *numRowsPtr, + const psArray *inputs) +{ + PS_ASSERT_ARRAY_NON_NULL(inputs, false); + PS_ASSERT_PTR_NON_NULL(minInputColsPtr, false); + PS_ASSERT_PTR_NON_NULL(maxInputColsPtr, false); + PS_ASSERT_PTR_NON_NULL(minInputRowsPtr, false); + PS_ASSERT_PTR_NON_NULL(maxInputRowsPtr, false); + PS_ASSERT_PTR_NON_NULL(numColsPtr, false); + PS_ASSERT_PTR_NON_NULL(numRowsPtr, false); + + // Step through each readout in the input image list to determine how big of an output image is needed to + // combine these input images. + int maxInputCols = 0; // The largest input column value + int maxInputRows = 0; // The largest input row value + int minInputCols = INT_MAX; // The smallest input column value + int minInputRows = INT_MAX; // The smallest input row value + int xSize = 0, ySize = 0; // The size of the output image + + int xMin = INT_MAX; + int yMin = INT_MAX; + int xMax = 0; + int yMax = 0; + + bool valid = false; // Do we have a single valid input? + for (long i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + + if (!readout) { + continue; + } + if (!readout->image) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Input readout %ld has NULL image.\n", i); + return false; + } + + // use the trimsec to define the max full range of the output pixels + pmCell *cell = readout->parent; // The parent cell + bool mdok = true; // Status of MD lookup + psRegion *trimsec = psMetadataLookupPtr(&mdok, cell->concepts, "CELL.TRIMSEC"); // Trim section + if (!mdok || !trimsec || psRegionIsNaN(*trimsec)) { + psWarning("CELL.TRIMSEC is not set for readout %ld --- ignored.\n", i); + } else { + xSize = PS_MAX(xSize, trimsec->x1 - trimsec->x0); + ySize = PS_MAX(ySize, trimsec->y1 - trimsec->y0); + xMin = PS_MIN(xMin, trimsec->x0); + xMax = PS_MAX(xMax, trimsec->x1); + yMin = PS_MIN(yMin, trimsec->y0); + yMax = PS_MAX(yMax, trimsec->y1); + } + + valid = true; + + // Range of pixels on output images + minInputCols = PS_MAX(xMin, PS_MIN(minInputCols, readout->col0)); + maxInputCols = PS_MIN(xMax, PS_MAX(maxInputCols, readout->col0 + readout->image->numCols)); + minInputRows = PS_MAX(yMin, PS_MIN(minInputRows, readout->row0)); + maxInputRows = PS_MIN(yMax, PS_MAX(maxInputRows, readout->row0 + readout->image->numRows)); + psTrace("psModules.camera", 7, "Readout %ld: offset %d,%d; size %dx%d\n", i, + readout->col0, readout->row0, readout->image->numCols, readout->image->numRows); + } + + if (minInputColsPtr) { + *minInputColsPtr = minInputCols; + } + if (maxInputColsPtr) { + *maxInputColsPtr = maxInputCols; + } + if (minInputRowsPtr) { + *minInputRowsPtr = minInputRows; + } + if (maxInputRowsPtr) { + *maxInputRowsPtr = maxInputRows; + } + if (numColsPtr) { + *numColsPtr = xSize; + } + if (numRowsPtr) { + *numRowsPtr = ySize; + } + + return valid; +} Index: /tags/pap_tags/pap_root_080320/psModules/src/camera/pmReadoutStack.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/camera/pmReadoutStack.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/camera/pmReadoutStack.h (revision 22293) @@ -0,0 +1,24 @@ +#ifndef PM_READOUT_STACK_H +#define PM_READOUT_STACK_H + +#include "pmHDU.h" +#include "pmFPA.h" + +/// Update an output readout (for a stack) with the correct col0,row0 and the image size +bool pmReadoutUpdateSize(pmReadout *readout, ///< Readout which to update + int minCols, int minRows, ///< Minimum coordinates + int numCols, int numRows, ///< Size of images + bool mask, ///< Worry about the mask? + bool weight, ///< Worry about the weight? + psMaskType blank ///< Mask value to give to blank pixels + ); + +/// Determine how large an output image is needed to combine the input readouts +bool pmReadoutStackValidate(int *minInputColsPtr, int *maxInputColsPtr, ///< Min and max size in x + int *minInputRowsPtr, int *maxInputRowsPtr, ///< Min and max size in y + int *numColsPtr, int *numRowsPtr, ///< Size of image + const psArray *inputs ///< Array of pmReadouts + ); + + +#endif Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/.cvsignore =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/.cvsignore (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/.cvsignore (revision 22293) @@ -0,0 +1,6 @@ +.deps +.libs +Makefile +Makefile.in +*.la +*.lo Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/Makefile.am =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/Makefile.am (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/Makefile.am (revision 22293) @@ -0,0 +1,35 @@ +noinst_LTLIBRARIES = libpsmodulesdetrend.la + +libpsmodulesdetrend_la_CPPFLAGS = $(SRCINC) $(PSMODULES_CFLAGS) +libpsmodulesdetrend_la_LDFLAGS = -release $(PACKAGE_VERSION) +libpsmodulesdetrend_la_SOURCES = \ + pmFlatField.c \ + pmFlatNormalize.c \ + pmFringeStats.c \ + pmMaskBadPixels.c \ + pmNonLinear.c \ + pmBias.c \ + pmOverscan.c \ + pmDetrendDB.c \ + pmShutterCorrection.c \ + pmShifts.c \ + pmDark.c + +# pmSkySubtract.c + +pkginclude_HEADERS = \ + pmFlatField.h \ + pmFlatNormalize.h \ + pmFringeStats.h \ + pmMaskBadPixels.h \ + pmNonLinear.h \ + pmBias.h \ + pmOverscan.h \ + pmDetrendDB.h \ + pmShutterCorrection.h \ + pmShifts.h \ + pmDark.h + +# pmSkySubtract.h + +CLEANFILES = *~ Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmBias.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmBias.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmBias.c (revision 22293) @@ -0,0 +1,194 @@ +#if HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmHDUUtils.h" +#include "pmFPALevel.h" +#include "pmFPAview.h" +#include "pmFPACalibration.h" + +#include "pmOverscan.h" +#include "pmBias.h" + +// pmBiasSubtractFrame(): this routine will take as input a readout for the input image and a readout for the bias +// image. The bias image is subtracted in place from the input image. +bool pmBiasSubtractFrame(pmReadout *in, // Input readout + const pmReadout *sub, // Readout to be subtracted from input + float scale // Scale to apply before subtracting + ) +{ + PS_ASSERT_PTR_NON_NULL(in, false); + PS_ASSERT_PTR_NON_NULL(in->image, false); + PS_ASSERT_IMAGE_TYPE(in->image, PS_TYPE_F32, false); + PS_ASSERT_IMAGE_NON_EMPTY(in->image, false); + PS_ASSERT_PTR_NON_NULL(sub, false); + PS_ASSERT_PTR_NON_NULL(sub->image, false); + PS_ASSERT_IMAGE_TYPE(sub->image, PS_TYPE_F32, false); + PS_ASSERT_IMAGE_NON_EMPTY(sub->image, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(in->image, sub->image, false); + + psImage *inImage = in->image; // The input image + psImage *inMask = in->mask; // The input mask + psImage *subImage = sub->image; // The image to be subtracted + psImage *subMask = sub->mask; // The mask for the subtraction image + + // Check parities + int xIpar = psMetadataLookupS32(NULL, in->parent->concepts, "CELL.XPARITY"); + int xSpar = psMetadataLookupS32(NULL, sub->parent->concepts, "CELL.XPARITY"); + if (xIpar != xSpar) { + psError(PS_ERR_UNKNOWN, true, "images for subtraction do not have the same " + "CELL.XPARITY (%d vs %d).\n pmSubtractBias must be upgraded to handle this situation\n", + xIpar, xSpar); + return false; + } + + int yIpar = psMetadataLookupS32(NULL, in->parent->concepts, "CELL.YPARITY"); + int ySpar = psMetadataLookupS32(NULL, sub->parent->concepts, "CELL.YPARITY"); + if (yIpar != ySpar) { + psError(PS_ERR_UNKNOWN, true, "images for subtraction do not have the same " + "CELL.YPARITY (%d vs %d).\n pmSubtractBias must be upgraded to handle this situation\n", + xIpar, xSpar); + return false; + } + + // Offsets of the cells + int x0in = psMetadataLookupS32(NULL, in->parent->concepts, "CELL.X0"); + int y0in = psMetadataLookupS32(NULL, in->parent->concepts, "CELL.Y0"); + int x0sub = psMetadataLookupS32(NULL, sub->parent->concepts, "CELL.X0"); + int y0sub = psMetadataLookupS32(NULL, sub->parent->concepts, "CELL.Y0"); + + if ((inImage->numCols + x0in - x0sub) > subImage->numCols) { + psError(PS_ERR_UNKNOWN, true, "Image does not have enough columns for subtraction (%d vs %d).\n", + inImage->numCols + x0in - x0sub, subImage->numCols); + return false; + } + if ((inImage->numRows + y0in - y0sub) > subImage->numRows) { + psError(PS_ERR_UNKNOWN, true, "Image does not have enough rows for subtraction (%d vs %d).\n", + inImage->numRows + y0in - y0sub, subImage->numRows); + return false; + } + + if (scale == 1.0) { + for (int i = 0; i < inImage->numRows; i++) { + for (int j = 0; j < inImage->numCols; j++) { + inImage->data.F32[i][j] -= subImage->data.F32[i+y0in-y0sub][j+x0in-x0sub]; + if (inMask && subMask) { + inMask->data.U8[i][j] |= subMask->data.U8[i+y0in-y0sub][j+x0in-x0sub]; + } + } + } + } else { + for (int i = 0; i < inImage->numRows; i++) { + for (int j = 0; j < inImage->numCols; j++) { + inImage->data.F32[i][j] -= subImage->data.F32[i+y0in-y0sub][j+x0in-x0sub] * scale; + if (inMask && subMask) { + inMask->data.U8[i][j] |= subMask->data.U8[i+y0in-y0sub][j+x0in-x0sub]; + } + } + } + } + + return true; +} + +bool pmBiasSubtract(pmReadout *in, pmOverscanOptions *overscanOpts, + const pmReadout *bias, const pmReadout *dark, const pmFPAview *view) +{ + psTrace("psModules.detrend", 4, + "---- pmBiasSubtract() begin ----\n"); + + PS_ASSERT_PTR_NON_NULL(in, false); + PS_ASSERT_IMAGE_NON_NULL(in->image, false); + PS_ASSERT_IMAGE_TYPE(in->image, PS_TYPE_F32, false); + if (bias) { + PS_ASSERT_IMAGE_NON_NULL(bias->image, false); + PS_ASSERT_IMAGE_TYPE(bias->image, PS_TYPE_F32, false); + } + if (dark) { + psWarning("Dark processing is now available using pmDark --- perhaps you should use that instead?"); + PS_ASSERT_PTR_NON_NULL(view, false); + PS_ASSERT_IMAGE_NON_NULL(dark->image, false); + PS_ASSERT_IMAGE_TYPE(dark->image, PS_TYPE_F32, false); + } + + pmHDU *hdu = pmHDUFromReadout(in); // HDU of interest + + if (!pmOverscanSubtract (in, overscanOpts)) { + return false; + } + + // Bias frame subtraction + if (bias) { + psVector *md5 = psImageMD5(bias->image); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psStringPrepend(&md5string, "BIAS image MD5: "); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + md5string, ""); + psFree(md5string); + + if (!pmBiasSubtractFrame(in, bias, 1.0)) { + return false; + } + } + + if (dark) { + // Get the scaling + float inTime = psMetadataLookupF32(NULL, in->parent->concepts, "CELL.DARKTIME"); + float darkTime = psMetadataLookupF32(NULL, dark->parent->concepts, "CELL.DARKTIME"); + if (isnan(inTime) || isnan(darkTime)) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine dark scaling."); + return false; + } + + float darkNorm = 1.0; + float inNorm = pmFPADarkNorm(in->parent->parent->parent, view, inTime); + + // if we have a normalized dark exposure, we simply multiply the master by inNorm. if + // we do not have a normalized exposure, we have to scale the master as well. XXX do + // we need to explicitly identify the master as normalized? + + if (darkTime != 1.0) { + darkNorm = pmFPADarkNorm(dark->parent->parent->parent, view, darkTime); + } + + if (isnan(inNorm) || isnan(darkNorm)) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine dark normalisations."); + return false; + } + + float scale = inNorm / darkNorm;// Scaling to apply to dark exposure + + psVector *md5 = psImageMD5(dark->image); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psStringPrepend(&md5string, "DARK image (scale %.3f) MD5: ", scale); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + md5string, ""); + psFree(md5string); + + if (!pmBiasSubtractFrame(in, dark, scale)) { + return false; + } + } + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now, used for reporting + psString timeString = psTimeToISO(time); // String with time + psFree(time); + psStringPrepend(&timeString, "Overscan/bias/dark processing completed at "); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + timeString, ""); + psFree(timeString); + + + return true; +} + + Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmBias.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmBias.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmBias.h (revision 22293) @@ -0,0 +1,37 @@ +/* @file pmBias.h + * @brief Subtract the overscan, bias and dark + * + * @author George Gusciora, MHPCC + * @author Paul Price, IfA + * + * @version $Revision: 1.7 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-08-15 20:21:18 $ + * Copyright 2004--2006 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_BIAS_H +#define PM_BIAS_H + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +/// Subtract the overscan, bias and/or dark +/// +/// Subtracts the overscan, as measured from the bias member of the input readout (if options are non-NULL), +/// bias (if non-NULL) and dark (if non-NULL) scaled by the CELL.DARKTIME concept. +bool pmBiasSubtract(pmReadout *in, ///< Input readout, to be overscan/bias/dark corrected + pmOverscanOptions *overscanOpts, ///< Options for overscan subtraction, or NULL + const pmReadout *bias, ///< Bias to subtract, or NULL + const pmReadout *dark, ///< Dark to scale and subtract, or NULL + const pmFPAview *view ///< View for readout of interest + ); + +// pmBiasSubtractFrame(): this routine will take as input a readout for the input image and a readout for the bias +// image. The bias image is subtracted in place from the input image. +bool pmBiasSubtractFrame(pmReadout *in, // Input readout + const pmReadout *sub, // Readout to be subtracted from input + float scale // Scale to apply before subtracting + ); + +/// @} +#endif Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmDark.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmDark.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmDark.c (revision 22293) @@ -0,0 +1,745 @@ +#include +#include +#include + +#include "psPolynomialMD.h" + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmHDUUtils.h" +#include "pmFPARead.h" +#include "pmFPAWrite.h" +#include "pmReadoutStack.h" + +#include "pmDark.h" + +#define PM_DARK_FITS_EXTNAME "PS_DARK" // FITS extension name for ordinates table +#define PM_DARK_FITS_NAME "NAME" // Column name for concept name in ordinates table +#define PM_DARK_FITS_ORDER "ORDER" // Column name for polynomial order in ordinates table +#define PM_DARK_FITS_SCALE "SCALE" // Column name for scaling option in ordinates table +#define PM_DARK_FITS_MIN "MIN" // Column name for minimum value in ordinates table +#define PM_DARK_FITS_MAX "MAX" // Column name for maximum value in ordinates table + + + +// Look up the value of an ordinate in a readout +static bool ordinateLookup(float *value, // Value of ordinate, to return + const char *name, // Name of ordinate (concept name) + bool scale, // Scale the value? + float min, float max, // Minimum and maximum values for scaling + const pmReadout *ro // Readout of interest + ) +{ + assert(value); + assert(name); + assert(ro); + + pmCell *cell = ro->parent; // Parent cell + if (!cell) { + return false; + } + psMetadataItem *item = psMetadataLookup(cell->concepts, name); + if (!item) { + pmChip *chip = cell->parent; // Parent chip + if (!chip) { + return false; + } + item = psMetadataLookup(chip->concepts, name); + if (!item) { + pmFPA *fpa = chip->parent; // Parent FPA + if (!fpa) { + return false; + } + item = psMetadataLookup(fpa->concepts, name); + if (!item) { + psWarning("Unable to find concept %s in readout", name); + return false; + } + } + } + + *value = psMetadataItemParseF32(item); // Value of interest + if (!isfinite(*value)) { + psWarning("Non-finite value (%f) of concept %s in readout", *value, name); + return false; + } + if (scale) { + if (*value < min || *value > max) { + psWarning("Value of concept %s (%f) outside range (%f:%f)", name, *value, min, max); + return false; + } + *value = 2.0 * (*value - min) / (max - min) - 1.0; + } + + return true; +} + +static void darkOrdinateFree(pmDarkOrdinate *ord) +{ + psFree(ord->name); + return; +} + +pmDarkOrdinate *pmDarkOrdinateAlloc(const char *name, int order) +{ + pmDarkOrdinate *ord = psAlloc(sizeof(pmDarkOrdinate)); // Ordinate data, to return + psMemSetDeallocator(ord, (psFreeFunc)darkOrdinateFree); + + ord->name = psStringCopy(name); + ord->order = order; + ord->scale = false; + ord->min = NAN; + ord->max = NAN; + + return ord; +} + + +bool pmDarkCombine(pmCell *output, const psArray *inputs, psArray *ordinates, const char *normConcept, + int iter, float rej, psMaskType maskVal) +{ + PS_ASSERT_PTR_NON_NULL(output, false); + PS_ASSERT_ARRAY_NON_NULL(inputs, false); + PS_ASSERT_ARRAY_NON_NULL(ordinates, false); + PS_ASSERT_INT_NONNEGATIVE(iter, false); + PS_ASSERT_FLOAT_LARGER_THAN(rej, 0.0, false); + + // Extract fitting orders + int numOrdinates = ordinates->n; // Number of ordinates + int numInputs = inputs->n; // Number of inputs + int numBadInputs = 0; // Number of bad inputs + psArray *values = psArrayAlloc(numInputs); + psVector *roMask = psVectorAlloc(numInputs, PS_TYPE_U8); // Mask for bad readouts + psVectorInit(roMask, 0); + psVector *norm = normConcept ? psVectorAlloc(numInputs, PS_TYPE_F32) : NULL; // Normalisations for each + for (int i = 0; i < numInputs; i++) { + values->data[i] = psVectorAlloc(numOrdinates, PS_TYPE_F32); + if (norm) { + pmReadout *ro = inputs->data[i]; // Readout of interest + float normValue; // Normalisation value + if (!ordinateLookup(&normValue, normConcept, false, NAN, NAN, ro)) { + psWarning("Unable to find value of %s for readout %d", normConcept, i); + roMask->data.U8[i] = 0xff; + norm->data.F32[i] = NAN; + numBadInputs++; + continue; + } + if (normValue == 0.0) { + psWarning("Normalisation value (%s) for readout %d is zero", normConcept, i); + roMask->data.U8[i] = 0xff; + norm->data.F32[i] = NAN; + numBadInputs++; + continue; + } + norm->data.F32[i] = 1.0 / normValue; + } + } + psVector *orders = psVectorAlloc(numOrdinates, PS_TYPE_U8); // Orders for each concept + for (int i = 0; i < numOrdinates; i++) { + pmDarkOrdinate *ord = ordinates->data[i]; // Ordinate information + if (ord->order <= 0) { + psError(PS_ERR_UNKNOWN, true, "Bad order for concept %s (%d) --- ignored", ord->name, ord->order); + psFree(values); + psFree(roMask); + psFree(orders); + return false; + } + orders->data.U8[i] = ord->order; + + // Mask the readout and move on + #define MASK_READOUT_VALUE { \ + roMask->data.U8[j] = 0xff; \ + val->data.F32[i] = NAN; \ + numBadInputs++; \ + continue; \ + } + + for (int j = 0; j < numInputs; j++) { + psVector *val = values->data[j]; // Value vector for readout + if (roMask->data.U8[j]) { + val->data.F32[i] = NAN; + continue; + } + + pmReadout *ro = inputs->data[j]; // Readout of interest + float value = NAN; // Value of ordinate + if (!ordinateLookup(&value, ord->name, ord->scale, ord->min, ord->max, ro)) { + roMask->data.U8[j] = 0xff; + val->data.F32[i] = NAN; + numBadInputs++; + continue; + } + val->data.F32[i] = value; + } + } + + if (psTraceGetLevel("psModules.detrend") > 9) { + for (int i = 0; i < numInputs; i++) { + psVector *val = values->data[i]; + (void) val; // avoid unused variable message when tracing is compiled out + for (int j = 0; j < numOrdinates; j++) { + psTrace("psModules.detrend", 9, "Image %d, ordinate %d: %f\n", i, j, val->data.F32[j]); + } + } + } + + psPolynomialMD *poly = psPolynomialMDAlloc(orders); // Polynomial for fitting + psFree(orders); + int numTerms = poly->coeff->n; // Number of terms in polynomial + if (numTerms > numInputs - numBadInputs) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Insufficient inputs (%d) to fit polynomial terms (%d).", + numInputs - numBadInputs, numTerms); + psFree(values); + psFree(roMask); + psFree(norm); + return false; + } + + // Set up output + int minInputCols, maxInputCols, minInputRows, maxInputRows; // Smallest and largest values to combine + int xSize, ySize; // Size of the output image + if (!pmReadoutStackValidate(&minInputCols, &maxInputCols, &minInputRows, &maxInputRows, &xSize, &ySize, + inputs)) { + psError(PS_ERR_UNKNOWN, false, "No valid input readouts."); + psFree(values); + psFree(roMask); + psFree(norm); + return false; + } + if (output->readouts->n != numTerms) { + output->readouts = psArrayRealloc(output->readouts, numTerms); + } + int outRow0 = 0, outCol0 = 0; // Output row0 and col0 + for (int i = 0; i < numTerms; i++) { + pmReadout *ro = output->readouts->data[i]; // Readout to update + if (!ro) { + ro = output->readouts->data[i] = pmReadoutAlloc(output); + } + + pmReadoutUpdateSize(ro, minInputCols, minInputRows, xSize, ySize, false, false, 0); + if (i == 0) { + outRow0 = ro->row0; + outCol0 = ro->col0; + } else { + assert(ro->row0 == outRow0); + assert(ro->col0 == outCol0); + } + } + + // Iterate over pixels, fitting polynomial + psVector *pixels = psVectorAlloc(numInputs, PS_TYPE_F32); // Stack of pixels + psVector *mask = psVectorAlloc(numInputs, PS_TYPE_MASK); // Mask for stack + for (int i = minInputRows; i < maxInputRows; i++) { + int yOut = i - outRow0; // y position on output readout +#ifdef SHOW_BUSY + if (psTraceGetLevel("psModules.detrend") > 9) { + printf("Processing row %d\r", i); + fflush(stdout); + } +#endif + + for (int j = minInputCols; j < maxInputCols; j++) { + int xOut = j - outCol0; // x position on output readout + + psVectorInit(mask, 0); + for (int r = 0; r < inputs->n; r++) { + if (roMask->data.U8[r]) { + mask->data.PS_TYPE_MASK_DATA[r] = 0xff; + continue; + } + pmReadout *readout = inputs->data[r]; // Input readout + int yIn = i - readout->row0; // y position on input readout + int xIn = j - readout->col0; // x position on input readout + + pixels->data.F32[r] = readout->image->data.F32[yIn][xIn]; + if (norm) { + pixels->data.F32[r] *= norm->data.F32[r]; + } + if (readout->mask) { + mask->data.PS_TYPE_MASK_DATA[r] = readout->mask->data.PS_TYPE_MASK_DATA[yIn][xIn]; + } + } + + if (!psPolynomialMDClipFit(poly, pixels, NULL, mask, maskVal, values, iter, rej)) { + psErrorClear(); // Nothing we can do about it + psVectorInit(poly->coeff, NAN); + } + for (int k = 0; k < numTerms; k++) { + pmReadout *ro = output->readouts->data[k]; // Readout of interest + ro->image->data.F32[yOut][xOut] = poly->coeff->data.F64[k]; + } + } + } + + psFree(norm); + psFree(roMask); + psFree(poly); + psFree(pixels); + psFree(mask); + psFree(values); + + psMetadataAddPtr(output->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_ORDINATES, + PS_DATA_ARRAY | PS_META_REPLACE, "Dark ordinates", ordinates); + psMetadataAddStr(output->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_NORM, + PS_META_REPLACE, "Dark normalisation", normConcept); + + for (int i = 0; i < numTerms; i++) { + pmReadout *ro = output->readouts->data[i]; // Readout to update + ro->data_exists = true; + } + output->data_exists = true; + output->parent->data_exists = true; + + return true; +} + + + +bool pmDarkApply(pmReadout *readout, const pmCell *dark, psMaskType bad) +{ + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_PTR_NON_NULL(dark, false); + PS_ASSERT_IMAGE_NON_NULL(readout->image, false); + PS_ASSERT_IMAGE_TYPE(readout->image, PS_TYPE_F32, false); + int numCols = readout->image->numCols, numRows = readout->image->numRows; // Size of image + if (readout->mask) { + PS_ASSERT_IMAGE_NON_NULL(readout->mask, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(readout->mask, readout->image, false); + PS_ASSERT_IMAGE_TYPE(readout->mask, PS_TYPE_MASK, false); + } + int numTerms = dark->readouts->n; // Number of polynomial terms + for (int i = 0; i < numTerms; i++) { + pmReadout *ro = dark->readouts->data[i]; // Dark readout + PS_ASSERT_PTR_NON_NULL(ro, false); + PS_ASSERT_IMAGE_NON_NULL(ro->image, false); + PS_ASSERT_IMAGE_SIZE(ro->image, numCols, numRows, false); + PS_ASSERT_IMAGE_TYPE(ro->image, PS_TYPE_F32, false); + } + psArray *ordinates = psMetadataLookupPtr(NULL, dark->analysis, PM_DARK_ANALYSIS_ORDINATES); // Ordinates + if (!ordinates) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find dark ordinates."); + return false; + } + bool mdok; // Status of MD lookup + psString normConcept = psMetadataLookupStr(&mdok, dark->analysis, PM_DARK_ANALYSIS_NORM); // Normalisation + + int numOrdinates = ordinates->n; // Number of ordinates + psVector *values = psVectorAlloc(numOrdinates, PS_TYPE_F32); // Values of ordinates + for (int i = 0; i < numOrdinates; i++) { + pmDarkOrdinate *ord = ordinates->data[i]; // Ordinate of interest + float value = NAN; // Value for ordinate + if (!ordinateLookup(&value, ord->name, ord->scale, ord->min, ord->max, readout)) { + psError(PS_ERR_UNKNOWN, true, "Unable to find value for %s", ord->name); + psFree(values); + return false; + } + values->data.F32[i] = value; + } + float norm = NAN; // Normalisation value + bool doNorm = false; // Do normalisation? + if (normConcept && strlen(normConcept) > 0) { + if (!ordinateLookup(&norm, normConcept, false, NAN, NAN, readout)) { + psError(PS_ERR_UNKNOWN, true, "Unable to find value for %s", normConcept); + psFree(values); + return false; + } + doNorm = true; + } + + psVector *orders = psVectorAlloc(numOrdinates, PS_TYPE_U8); // Order for each polynomial + for (int i = 0; i < numOrdinates; i++) { + pmDarkOrdinate *ord = ordinates->data[i]; // Ordinate information + if (ord->order <= 0) { + psError(PS_ERR_UNKNOWN, true, "Bad order for concept %s (%d) --- ignored", ord->name, ord->order); + psFree(values); + psFree(orders); + return false; + } + orders->data.U8[i] = ord->order; + } + + psPolynomialMD *poly = psPolynomialMDAlloc(orders); // Polynomial to apply + psFree(orders); + + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + for (int i = 0; i < numTerms; i++) { + pmReadout *ro = dark->readouts->data[i]; // Dark readout + poly->coeff->data.F64[i] = ro->image->data.F32[y][x]; + } + float value = psPolynomialMDEval(poly, values); // Value of dark current + if (doNorm) { + value *= norm; + } + readout->image->data.F32[y][x] -= value; + if (readout->mask && !isfinite(value)) { + readout->mask->data.PS_TYPE_MASK_DATA[y][x] = bad; + } + } + } + + psFree(poly); + psFree(values); + + return true; +} + +bool pmFPAWriteDark(pmFPA *fpa, psFits *fits, psDB *db, bool blank, bool recurse) +{ + PS_ASSERT_PTR_NON_NULL(fpa, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + if (!pmFPAWrite(fpa, fits, db, blank, recurse)) { + psError(PS_ERR_IO, false, "Unable to write FPA dark images"); + return false; + } + + pmHDU *hdu = fpa->hdu; + if (blank || !hdu) { + // No more to do + return true; + } + + psArray *ordinates = NULL; // Dark ordinates, to write + const char *normConcept = NULL; // Normalisation concept + psArray *chips = fpa->chips; // Component chips + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + psArray *cells = chip->cells; // Component cells + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; // Cell of interest + bool mdok; // Status of MD lookup + psArray *newOrd = psMetadataLookupPtr(&mdok, cell->analysis, PM_DARK_ANALYSIS_ORDINATES); // Ords + if (!mdok) { + continue; + } + psString newNorm = psMetadataLookupPtr(&mdok, cell->analysis, PM_DARK_ANALYSIS_NORM); // Norm + if (ordinates) { + if (newOrd != ordinates) { + psError(PS_ERR_UNKNOWN, true, "Dark ordinates differ across cells."); + return false; + } + if ((normConcept && (!newNorm || strcmp(normConcept, newNorm) != 0)) || + (!normConcept && newNorm)) { + psError(PS_ERR_UNKNOWN, true, "Dark normalisations differ across cells."); + return false; + } + } else { + ordinates = newOrd; + normConcept = newNorm; + } + } + } + + if (!ordinates) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find dark ordinates."); + return false; + } + + return pmDarkWrite(fits, hdu->header, ordinates, normConcept); +} + +bool pmChipWriteDark(pmChip *chip, psFits *fits, psDB *db, bool blank, bool recurse) +{ + PS_ASSERT_PTR_NON_NULL(chip, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + if (!pmChipWrite(chip, fits, db, blank, recurse)) { + psError(PS_ERR_IO, false, "Unable to write chip dark images"); + return false; + } + + pmHDU *hdu = pmHDUFromChip(chip); + if (blank || !hdu) { + // No more to do + return true; + } + + psArray *ordinates = NULL; // Dark ordinates, to write + const char *normConcept = NULL; // Normalisation concept + psArray *cells = chip->cells; // Component cells + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; // Cell of interest + bool mdok; // Status of MD lookup + psArray *newOrd = psMetadataLookupPtr(&mdok, cell->analysis, PM_DARK_ANALYSIS_ORDINATES); // Ordinates + if (!mdok) { + continue; + } + psString newNorm = psMetadataLookupPtr(&mdok, cell->analysis, PM_DARK_ANALYSIS_NORM); // Normalisation + if (ordinates) { + if (newOrd != ordinates) { + psError(PS_ERR_UNKNOWN, true, "Dark ordinates differ across cells."); + return false; + } + if ((normConcept && (!newNorm || strcmp(normConcept, newNorm) != 0)) || + (!normConcept && newNorm)) { + psError(PS_ERR_UNKNOWN, true, "Dark normalisations differ across cells."); + return false; + } + } else { + ordinates = newOrd; + normConcept = newNorm; + } + } + + if (!ordinates) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find dark ordinates."); + return false; + } + + return pmDarkWrite(fits, hdu->header, ordinates, normConcept); +} + +bool pmCellWriteDark(pmCell *cell, psFits *fits, psDB *db, bool blank) +{ + PS_ASSERT_PTR_NON_NULL(cell, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + // Allow the usual pmFPAWrite functions to handle the heavy lifting for the images + if (!pmCellWrite(cell, fits, db, blank)) { + psError(PS_ERR_UNKNOWN, false, "Unable to write dark cell."); + return false; + } + + pmHDU *hdu = pmHDUFromCell(cell); + if (blank || !hdu) { + // No more to do + return true; + } + + psArray *ordinates = psMetadataLookupPtr(NULL, cell->analysis, PM_DARK_ANALYSIS_ORDINATES); + if (!ordinates) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find dark ordinates."); + return false; + } + bool mdok; // Status of MD lookup + psString normConcept = psMetadataLookupPtr(&mdok, cell->analysis, PM_DARK_ANALYSIS_NORM); // Normalisation + + return pmDarkWrite(fits, hdu->header, ordinates, normConcept); +} + +bool pmDarkWrite(psFits *fits, psMetadata *header, const psArray *ordinates, const char *normConcept) +{ + PS_ASSERT_FITS_NON_NULL(fits, false); + PS_ASSERT_FITS_WRITABLE(fits, false); + PS_ASSERT_ARRAY_NON_NULL(ordinates, false); + + // Only write table once per FITS file + bool write = false; // Write table? + if (psFitsGetSize(fits) <= 1) { + write = true; + } else { + psMetadata *headers = psFitsReadHeaderSet(NULL, fits); // FITS headers + if (!psMetadataLookup(headers, PM_DARK_FITS_EXTNAME)) { + write = true; + } + psFree(headers); + } + if (!write) { + return true; + } + + if (!psMemIncrRefCounter((psMetadata*)header)) { + header = psMetadataAlloc(); + } + psMetadataAddStr(header, PS_LIST_TAIL, PM_DARK_HEADER_NORM, PS_META_REPLACE, + "Dark normalisation concept", normConcept); + + if (ordinates->n > 0) { + // Format ordinates into FITS table + int numOrdinates = ordinates->n;// Number of ordinates + psArray *table = psArrayAlloc(numOrdinates); // FITS table, constructed from ordinates + for (int i = 0; i < ordinates->n; i++) { + pmDarkOrdinate *ord = ordinates->data[i]; // Ordinate of interest + psMetadata *row = psMetadataAlloc(); // FITS table row + psMetadataAddStr(row, PS_LIST_TAIL, PM_DARK_FITS_NAME, 0, "Concept name", ord->name); + psMetadataAddS32(row, PS_LIST_TAIL, PM_DARK_FITS_ORDER, 0, "Polynomial order", ord->order); + psMetadataAddBool(row, PS_LIST_TAIL, PM_DARK_FITS_SCALE, 0, "Scale values?", ord->scale); + psMetadataAddF32(row, PS_LIST_TAIL, PM_DARK_FITS_MIN, 0, "Minimum value", ord->min); + psMetadataAddF32(row, PS_LIST_TAIL, PM_DARK_FITS_MAX, 0, "Maximum value", ord->max); + table->data[i] = row; + } + + if (!psFitsWriteTable(fits, header, table, PM_DARK_FITS_EXTNAME)) { + psError(PS_ERR_IO, false, "Unable to write dark ordinates."); + psFree(table); + psFree(header); + return false; + } + psFree(table); + } else { + // No ordinates to write to a table, so write to a blank header. + if (!psFitsWriteBlank(fits, header, PM_DARK_FITS_EXTNAME)) { + psError(PS_ERR_IO, false, "Unable to write dark header."); + psFree(header); + return false; + } + } + psFree(header); + + return true; +} + + +bool pmFPAReadDark(pmFPA *fpa, psFits *fits, psDB *db) +{ + PS_ASSERT_PTR_NON_NULL(fpa, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + if (!pmFPARead(fpa, fits, db)) { + psError(PS_ERR_UNKNOWN, false, "Unable to read dark FPA."); + return false; + } + + psString normConcept = NULL; // Normalisation concept + psArray *ordinates = pmDarkRead(&normConcept, fits); // Dark ordinates + if (!ordinates) { + psError(PS_ERR_IO, false, "Unable to read dark ordinates."); + return false; + } + + psArray *chips = fpa->chips; // Component chips + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + psArray *cells = chip->cells; // Component cells + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; // Cell of interest + psMetadataAddPtr(cell->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_ORDINATES, + PS_DATA_ARRAY | PS_META_REPLACE, "Dark ordinates", ordinates); + psMetadataAddStr(cell->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_NORM, PS_META_REPLACE, + "Dark normalisation", normConcept); + } + } + psFree(ordinates); + psFree(normConcept); + + return true; +} + +bool pmChipReadDark(pmChip *chip, psFits *fits, psDB *db) +{ + PS_ASSERT_PTR_NON_NULL(chip, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + if (!pmChipRead(chip, fits, db)) { + psError(PS_ERR_UNKNOWN, false, "Unable to read dark chip."); + return false; + } + + psString normConcept = NULL; // Normalisation concept + psArray *ordinates = pmDarkRead(&normConcept, fits); // Dark ordinates + if (!ordinates) { + psError(PS_ERR_IO, false, "Unable to read dark ordinates."); + return false; + } + + psArray *cells = chip->cells; // Component cells + for (int i = 0; i < cells->n; i++) { + pmCell *cell = cells->data[i]; // Cell of interest + psMetadataAddPtr(cell->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_ORDINATES, + PS_DATA_ARRAY | PS_META_REPLACE, "Dark ordinates", ordinates); + psMetadataAddStr(cell->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_NORM, PS_META_REPLACE, + "Dark normalisation", normConcept); + } + psFree(ordinates); + psFree(normConcept); + + return true; +} + + +bool pmCellReadDark(pmCell *cell, psFits *fits, psDB *db) +{ + PS_ASSERT_PTR_NON_NULL(cell, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + // Allow the usual pmFPARead functions to handle the heavy lifting for the images + if (!pmCellRead(cell, fits, db)) { + psError(PS_ERR_UNKNOWN, false, "Unable to read dark cell."); + return false; + } + + psString normConcept = NULL; // Normalisation concept + psArray *ordinates = pmDarkRead(&normConcept, fits); // Dark ordinates + if (!ordinates) { + psError(PS_ERR_IO, false, "Unable to read dark ordinates."); + return false; + } + psMetadataAddPtr(cell->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_ORDINATES, + PS_DATA_ARRAY | PS_META_REPLACE, "Dark ordinates", ordinates); + psMetadataAddStr(cell->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_NORM, PS_META_REPLACE, + "Dark normalisation", normConcept); + psFree(ordinates); + psFree(normConcept); + + return true; +} + +psArray *pmDarkRead(psString *normConcept, psFits *fits) +{ + PS_ASSERT_PTR_NON_NULL(normConcept, NULL); + PS_ASSERT_PTR_NULL(*normConcept, NULL); + PS_ASSERT_FITS_NON_NULL(fits, NULL); + + if (!psFitsMoveExtName(fits, PM_DARK_FITS_EXTNAME)) { + psError(PS_ERR_IO, false, "Unable to find extension with dark ordinates table (%s).", + PM_DARK_FITS_EXTNAME); + return false; + } + + psMetadata *header = psFitsReadHeader(NULL, fits); // Header + bool mdok; // Status of MD lookup + *normConcept = psMemIncrRefCounter(psMetadataLookupStr(&mdok, header, PM_DARK_HEADER_NORM)); + + psArray *ordinates = NULL; // Dark ordinates to return + + psFitsType type = psFitsGetExtType(fits); // Type of FITS extension + switch (type) { + case PS_FITS_TYPE_IMAGE: { + // Check that it's of zero size; otherwise we might have some conflict + int numCols = psMetadataLookupS32(&mdok, header, "NAXIS1"); + int numRows = psMetadataLookupS32(&mdok, header, "NAXIS2"); + if (numCols != 0 || numRows != 0) { + psError(PS_ERR_UNKNOWN, true, "Extension %s is not a DARK table.", PM_DARK_FITS_EXTNAME); + psFree(header); + return NULL; + } + // No ordinates fit --- only a constant term + ordinates = psArrayAlloc(0); + break; + } + case PS_FITS_TYPE_BINARY_TABLE: + case PS_FITS_TYPE_ASCII_TABLE: { + psArray *table = psFitsReadTable(fits); // FITS Table with ordinates + int numOrdinates = table->n; // Number of ordinates + ordinates = psArrayAlloc(numOrdinates); + + for (int i = 0; i < numOrdinates; i++) { + psMetadata *row = table->data[i]; // Row of interest + const char *name = psMetadataLookupStr(NULL, row, PM_DARK_FITS_NAME); // Concept name + int order = psMetadataLookupS32(NULL, row, PM_DARK_FITS_ORDER); // Polynomial order + if (!name || order <= 0) { + psError(PS_ERR_UNKNOWN, false, "Bad value reading dark ordinates table."); + psFree(table); + psFree(ordinates); + return false; + } + pmDarkOrdinate *ord = pmDarkOrdinateAlloc(name, order); // Ordinate data + ord->scale = psMetadataLookupBool(NULL, row, PM_DARK_FITS_SCALE); + ord->min = psMetadataLookupF32(NULL, row, PM_DARK_FITS_MIN); + ord->max = psMetadataLookupF32(NULL, row, PM_DARK_FITS_MAX); + + ordinates->data[i] = ord; + } + psFree(table); + break; + } + default: + psError(PS_ERR_UNKNOWN, true, "Unrecognised FITS extension type."); + return NULL; + } + psFree(header); + + return ordinates; +} + Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmDark.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmDark.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmDark.h (revision 22293) @@ -0,0 +1,98 @@ +#ifndef PM_DARK_H +#define PM_DARK_H + +#include +#include +#include + +#define PM_DARK_ANALYSIS_ORDINATES "DARK.ORDINATES" // Name for dark ordinates in the cell analysis metadata +#define PM_DARK_ANALYSIS_NORM "DARK.NORM" // Name for dark normalisation concept in cell analysis metadata +#define PM_DARK_HEADER_NORM "PSDRKNRM" // Header keyword for dark normalisation concept + +// An ordinate for fitting darks +typedef struct { + psString name; // Name of concept to fit + int order; // Polynomial order to fit + bool scale; // Rescale values? + float min, max; // Minimum and maximum values for rescaling +} pmDarkOrdinate; + +// Allocator +pmDarkOrdinate *pmDarkOrdinateAlloc(const char *name, // Name for ordinate + int order // Order for ordinate + ); + + +// Combine darks +bool pmDarkCombine(pmCell *output, // Output cell; readouts will be attached + const psArray *inputs, // Input readouts for combination + psArray *ordinates, // Ordinates for fitting + const char *normConcept, // Concept name to use to divide input pixel values + int iter, // Number of rejection iterations + float rej, // Rejection threshold (standard deviations) + psMaskType maskVal // Value to mask + ); + +// Apply dark +bool pmDarkApply(pmReadout *readout, // Readout to which to apply dark + const pmCell *dark, // Dark to apply + psMaskType bad // Mask value to give bad pixels + ); + +// I/O functions for darks + +// Write all darks within an FPA +bool pmFPAWriteDark(pmFPA *fpa, // FPA to write + psFits *fits, // FITS file to which to write + psDB *db, // Database, for concepts + bool blank, // Write a blank only? + bool recurse // Recurse to lower levels? + ); + +// Write all darks within a chip +bool pmChipWriteDark(pmChip *chip, // Chip to write + psFits *fits, // FITS file to which to write + psDB *db, // Database, for concepts + bool blank, // Write a blank only? + bool recurse // Recurse to lower levels? + ); + +// Write a dark to a FITS file +bool pmCellWriteDark(pmCell *cell, // Cell containing dark information + psFits *fits, // FITS file to which to write + psDB *db, // Database, for concepts + bool blank // Write a blank only? + ); + +// Read dark for all FPA from a FITS file +bool pmFPAReadDark(pmFPA *fpa, // FPA for which to read + psFits *fits, // FITS file to read + psDB *db // Database, for concepts + ); + +// Read dark for all chip from a FITS file +bool pmChipReadDark(pmChip *chip, // Chip for which to read + psFits *fits, // FITS file to read + psDB *db // Database, for concepts + ); + +// Read dark for a cell from a FITS file +bool pmCellReadDark(pmCell *cell, // Cell for which to read + psFits *fits, // FITS file to read + psDB *db // Database, for concepts + ); + +// Write dark table to FITS file +bool pmDarkWrite(psFits *fits, // FITS file to which to write + psMetadata *header, // Header to write + const psArray *ordinates, // Dark ordinates to write + const char *normConcept // Normalisation concept name + ); + +// Read dark table from FITS file +psArray *pmDarkRead(psString *normConcept, // Normalisation concept name + psFits *fits // FITS file to read + ); + + +#endif Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmDetrendDB.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmDetrendDB.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmDetrendDB.c (revision 22293) @@ -0,0 +1,335 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pmConfig.h" +#include "pmConfigCommand.h" +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmFPALevel.h" +#include "pmDetrendDB.h" +#include "psPipe.h" +#include "psIOBuffer.h" + +// ************* detrend select functions ************** + +// +static void pmDetrendSelectOptionsFree (pmDetrendSelectOptions *options) +{ + + if (!options) + return; + + psFree (options->camera); + psFree (options->filter); + psFree (options->dettype); + psFree (options->version); + + return; +} + +// define basic options for a new detrend database query +pmDetrendSelectOptions *pmDetrendSelectOptionsAlloc(const char *camera, psTime time, pmDetrendType type) +{ + + pmDetrendSelectOptions *options = psAlloc(sizeof(pmDetrendSelectOptions)); + psMemSetDeallocator(options, (psFreeFunc) pmDetrendSelectOptionsFree); + + // basic options required by every query + options->camera = psStringCopy (camera); + options->time = time; + options->type = type; + + // these other options depend on the type of detrend data + options->filter = NULL; + options->version = NULL; + options->dettype = NULL; + options->exptime = 0.0; + options->airmass = 0.0; + options->dettemp = 0.0; + options->twilight = 0.0; + + options->exptimeSet = false; // not selected + options->airmassSet = false; // not selected + options->dettempSet = false; // not selected + options->twilightSet = false; // not selected + + return options; +} + +static void pmDetrendSelectResultsFree (pmDetrendSelectResults *results) +{ + + if (!results) + return; + + psFree (results->detID); + psFree(results->level); + + return; +} + +pmDetrendSelectResults *pmDetrendSelectResultsAlloc () +{ + + pmDetrendSelectResults *results = psAlloc(sizeof(pmDetrendSelectResults)); + psMemSetDeallocator(results, (psFreeFunc) pmDetrendSelectResultsFree); + + results->detID = NULL; + results->level = NULL; + + return results; +} + +psString pmDetrendTypeToString (pmDetrendType type) +{ + + # define DETREND_STRING_CASE(TTT) \ +case PM_DETREND_TYPE_##TTT: \ + return psStringCopy (#TTT); + + switch (type) { + DETREND_STRING_CASE (MASK); + DETREND_STRING_CASE (BIAS); + DETREND_STRING_CASE (DARK); + DETREND_STRING_CASE (FLAT); + DETREND_STRING_CASE (FLAT_CORRECTION); + DETREND_STRING_CASE (SHUTTER); + DETREND_STRING_CASE (FRINGE); + DETREND_STRING_CASE (ASTROM); + default: + return NULL; + } + return NULL; +} + +// detselect -camera (camera) -time (time) -type (type) [others] +// returns: (type) (class) (exp_flag) DONE +pmDetrendSelectResults *pmDetrendSelect (const pmDetrendSelectOptions *options, + const pmConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(options, NULL); + PS_ASSERT_PTR_NON_NULL(config, NULL); + + int status, exit_status; + psString line = NULL; + psString time = psTimeToISO (&options->time); + + char *type = NULL; + if (options->dettype) { + type = psMemIncrRefCounter (options->dettype); + } else { + type = pmDetrendTypeToString (options->type); + } + unsigned int nFail; + + pmDetrendSelectResults *results = pmDetrendSelectResultsAlloc(); + psStringAppend(&line, "detselect -search -inst %s -det_type %s -time %s", options->camera, type, time); + + if (options->filter) { + psStringAppend(&line, " -filter %s", options->filter); + } + if (options->version) { + psStringAppend(&line, " -version %s", options->version); + } + if (options->exptimeSet) { + psStringAppend(&line, " -exp_time %f", options->exptime); + } + if (options->airmassSet) { + psStringAppend(&line, " -airmass %f", options->airmass); + } + if (options->dettempSet) { + psStringAppend(&line, " -airmass %f", options->dettemp); + } + if (options->twilightSet) { + psStringAppend(&line, " -airmass %f", options->twilight); + } + + psIOBuffer *buffer = NULL; + psPipe *pipe = NULL; + psMetadata *answer = NULL; + + if (!pmConfigDatabaseCommand(&line, config)) { + psError (PS_ERR_IO, false, "error building detrend command %s", line); + goto failure; + } + + if (!pmConfigTraceCommand(&line)) { + psError (PS_ERR_IO, false, "error building detrend command %s", line); + goto failure; + } + + psTrace("psModules.detrend", 5, "running %s", line); + + // use psPipe to exec the command, wait for response + buffer = psIOBufferAlloc (512); + pipe = psPipeOpen (line); + if (!pipe) { + psError (PS_ERR_IO, false, "error calling command %s", line); + goto failure; + } + + status = psIOBufferReadEmpty (buffer, 2000, pipe->fd_stdout); + if (!status) { + psError (PS_ERR_IO, false, "detselect is not responding"); + goto failure; + } + exit_status = psPipeClose (pipe); + if (exit_status) { + psError (PS_ERR_IO, false, "error running detselect"); + goto failure; + } + + psTrace("psModules.detrend", 5, "got answer: %s\n", buffer->data); + + nFail = 0; + answer = psMetadataConfigParse (NULL, &nFail, buffer->data, false); + if (!answer) { + psError(PS_ERR_IO, false, "failed to parse response from detselect\n"); + psLogMsg ("psModule.detrend", PS_LOG_ERROR, "detselect response (%d bytes):\n %s\n", buffer->n, buffer->data); + psLogMsg ("psModule.detrend", PS_LOG_ERROR, "detselect command:\n %s\n", line); + goto failure; + } + + psMetadata *md = psMetadataLookupPtr (NULL, answer, "detExp"); + if (!md) { + psError(PS_ERR_IO, false, "detselect response is missing 'detExp' Metadata\n"); + psLogMsg ("psModule.detrend", PS_LOG_ERROR, "detselect response:\n %s\n", buffer->data); + goto failure; + } + + bool mdstatus; + int detID = psMetadataLookupS32 (&mdstatus, md, "det_id"); + if (!mdstatus) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find det_id in output from detselect."); + psLogMsg ("psModule.detrend", PS_LOG_ERROR, "detselect response:\n %s\n", buffer->data); + goto failure; + } + int iteration = psMetadataLookupS32 (&mdstatus, md, "iteration"); + if (!mdstatus) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find iteration in output from detselect."); + psLogMsg ("psModule.detrend", PS_LOG_ERROR, "detselect response:\n %s\n", buffer->data); + goto failure; + } + psString fileLevel = psMetadataLookupStr(&mdstatus, md, "filelevel"); + if (!mdstatus || !fileLevel || strlen(fileLevel) == 0) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find filelevel in output from detselect."); + psLogMsg ("psModule.detrend", PS_LOG_ERROR, "detselect response:\n %s\n", buffer->data); + goto failure; + } + + results->detID = NULL; // it should be NULL already from the Alloc above + psStringAppend (&results->detID, " -det_id %d -iteration %d ", detID, iteration); + results->level = psMemIncrRefCounter(fileLevel); + + psTrace("psModules.detrend", 5, "generated detID %s\n", results->detID); + + psFree (answer); + psFree (buffer); + psFree (pipe); + psFree (line); + psFree (time); + psFree (type); + return results; + +failure: + psFree (answer); + psFree (results); + psFree (pipe); + psFree (buffer); + psFree (line); + psFree (type); + psFree (time); + return NULL; +} + +// ************* detrend file functions ************** + +// detselect -select -detID (detID) -classID (classID) +// returns: (detID) (classID) (filename) DONE +char *pmDetrendFile (const char *detID, const char *classID, const pmConfig *config) +{ + unsigned int nFail; + + PS_ASSERT_PTR_NON_NULL(detID, NULL); + + bool status; + psString line = NULL; + psArray *array = NULL; + + // generate the detselect command + psStringAppend (&line, "detselect -select %s", detID); + if (classID && strlen(classID) > 0) { + psStringAppend(&line, " -class_id %s", classID); + } + pmConfigDatabaseCommand(&line, config); + pmConfigTraceCommand(&line); + psTrace("psModules.detrend", 5, "running %s", line); + + // use psPipe to exec the command, wait for response + psIOBuffer *buffer = psIOBufferAlloc (512); + psPipe *pipe = psPipeOpen (line); + if (!pipe) { + psError (PS_ERR_IO, false, "error calling command %s", line); + goto failure; + } + + // timeout somewhat longer than 2sec. this could still be too short.... + status = psIOBufferReadEmpty (buffer, 2000, pipe->fd_stdout); + if (!status) { + psError (PS_ERR_IO, false, "detselect is not responding"); + goto failure; + } + status = psPipeClose (pipe); + if (status) { + psError (PS_ERR_IO, false, "error running detselect"); + goto failure; + } + + psTrace("psModules.detrend", 5, "got answer: %s\n", buffer->data); + + if (!buffer->n) { + psLogMsg ("psModule.detrend", PS_LOG_ERROR, "detselect response (%d bytes):\n %s\n", buffer->n, buffer->data); + psError(PS_ERR_IO, true, "no matching detrend data in database\n"); + goto failure; + } + + psMetadata *answer = psMetadataConfigParse (NULL, &nFail, buffer->data, false); + if (!answer) { + psError(PS_ERR_IO, false, "failed to parse response from detselect\n"); + psLogMsg ("psModule.detrend", PS_LOG_ERROR, "detselect response (%d bytes):\n %s\n", buffer->n, buffer->data); + goto failure; + } + psMetadataItem *item = psMetadataLookup (answer, "detNormalizedImfile"); + if ((item->type == PS_DATA_METADATA_MULTI) && (item->data.list->n > 1)) { + psError(PS_ERR_IO, false, "detselect returned too many files\n"); + goto failure; + } + + psMetadata *md = psMetadataLookupPtr (NULL, answer, "detNormalizedImfile"); + if (!md) { + psError(PS_ERR_IO, false, "detselect response is missing 'detNormalizedImfile' Metadata\n"); + psLogMsg ("psModule.detrend", PS_LOG_ERROR, "detselect response:\n %s\n", buffer->data); + goto failure; + } + + char *result = psStringCopy (psMetadataLookupStr (NULL, md, "uri")); + psTrace("psModules.detrend", 5, "detrend file: %s\n", result); + + psFree (answer); + psFree (pipe); + psFree (buffer); + psFree (line); + return result; + +failure: + psFree (array); + psFree (pipe); + psFree (buffer); + psFree (line); + return NULL; +} Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmDetrendDB.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmDetrendDB.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmDetrendDB.h (revision 22293) @@ -0,0 +1,65 @@ +/* @file pmDetrendDB.h + * @brief Tools to query the detrend database system + * + * the functions in here do not perform the detrend database queries directly. all interfaces + * to the detrend database go through the external dettools functions. this allows the modules + * and directly dependent program to be sufficiently independent of the database schema that it + * can be used with any properly defined detrend database tables. + * + * @author EAM, IfA + * + * @version $Revision: 1.14 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-11-10 01:09:20 $ + * Copyright 2004-2005 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_DETREND_DB_H +#define PM_DETREND_DB_H + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +typedef enum { + PM_DETREND_TYPE_MASK, + PM_DETREND_TYPE_BIAS, + PM_DETREND_TYPE_DARK, + PM_DETREND_TYPE_FLAT, + PM_DETREND_TYPE_FLAT_CORRECTION, + PM_DETREND_TYPE_SHUTTER, + PM_DETREND_TYPE_FRINGE, + PM_DETREND_TYPE_BACKGROUND, + PM_DETREND_TYPE_ASTROM, +} pmDetrendType; + +typedef struct { + char *camera; // name of camera + char *version; // optional version string + char *filter; // name of filter + char *dettype; // actual detrend type name + float exptime; // exposure time (for dark, maybe flat & fringe) + float airmass; // for fringe + float dettemp; // for fringe + float twilight; // hours (or seconds?) since/before nearest twilight + psTime time; // time of input data + pmDetrendType type; // type of detrend data + + bool exptimeSet; + bool airmassSet; + bool dettempSet; + bool twilightSet; +} pmDetrendSelectOptions; + +typedef struct { + char *detID; // identifier of detrend run + psString level; // level in FPA hierarchy of individual file +} pmDetrendSelectResults; + +psString pmDetrendTypeToString (pmDetrendType type); + +pmDetrendSelectOptions *pmDetrendSelectOptionsAlloc(const char *camera, psTime time, pmDetrendType type); +pmDetrendSelectResults *pmDetrendSelectResultsAlloc(); +pmDetrendSelectResults *pmDetrendSelect (const pmDetrendSelectOptions *options, const pmConfig *config); +char *pmDetrendFile (const char *detID, const char *classID, const pmConfig *config); + +/// @} +# endif Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFlatField.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFlatField.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFlatField.c (revision 22293) @@ -0,0 +1,119 @@ +#if HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmHDUUtils.h" +#include "pmFPAMaskWeight.h" +#include "pmFlatField.h" + +bool pmFlatField(pmReadout *in, const pmReadout *flat, psMaskType badFlat) +{ + PS_ASSERT_PTR_NON_NULL(in, false); + PS_ASSERT_PTR_NON_NULL(in->image, false); + PS_ASSERT_IMAGE_NON_EMPTY(in->image, false); + PS_ASSERT_PTR_NON_NULL(flat, false); + PS_ASSERT_PTR_NON_NULL(flat->image, false); + PS_ASSERT_IMAGE_NON_EMPTY(flat->image, false); + if (in->mask) { + PS_ASSERT_IMAGE_TYPE(in->mask, PS_TYPE_MASK, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(in->mask, in->image, false); + } + PS_ASSERT_IMAGE_TYPE(flat->image, in->image->type.type, false); + if (flat->mask) { + PS_ASSERT_IMAGE_TYPE(flat->mask, PS_TYPE_MASK, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(flat->mask, flat->image, false); + } + + psImage *inImage = in->image; // Input image + psImage *inMask = in->mask; // Mask for input image + psImage *flatImage = flat->image; // Flat-field image + psImage *flatMask = flat->mask; // Mask for flat-field image + + // Add flat-field MD5 to header + pmHDU *hdu = pmHDUFromReadout(in); // HDU of interest + psVector *md5 = psImageMD5(flat->image); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psStringPrepend(&md5string, "FLAT image MD5: "); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + md5string, ""); + psFree(md5string); + + // Check input image is not larger than flat image; mask is the same size as the input + if (inImage->numRows > flatImage->numRows || inImage->numCols > flatImage->numCols) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, "Input image (%dx%d) is larger than flat-field image " + "(%dx%d).\n", inImage->numCols, inImage->numRows, flatImage->numCols, flatImage->numRows); + return false; + } + + // Offsets on the chip + int x0in = psMetadataLookupS32(NULL, in->parent->concepts, "CELL.X0"); + int y0in = psMetadataLookupS32(NULL, in->parent->concepts, "CELL.Y0"); + int x0flat = psMetadataLookupS32(NULL, flat->parent->concepts, "CELL.X0"); + int y0flat = psMetadataLookupS32(NULL, flat->parent->concepts, "CELL.Y0"); + + // Determine offset based on image offset with chip offset: input frame to flat frame + int yOffset = in->row0 + y0in - flat->row0 - y0flat; + int xOffset = in->col0 + x0in - flat->col0 - x0flat; + + // Check that offsets are within image limits + if (inImage->numRows + yOffset > flatImage->numRows || + inImage->numCols + xOffset > flatImage->numCols) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, "Input image (%dx%d) with offsets (%d,%d) is larger than " + "flat-field image (%dx%d).\n", inImage->numCols, inImage->numRows, xOffset, yOffset, + flatImage->numCols, flatImage->numRows); + return false; + } + + // Macro for all PS types + #define FLAT_DIVISION_CASE(TYPE, SPECIAL) \ +case PS_TYPE_##TYPE: \ + for (int j = 0; j < inImage->numRows; j++) { \ + for (int i = 0; i < inImage->numCols; i++) { \ + ps##TYPE flatValue = flatImage->data.TYPE[j + yOffset][i + xOffset]; \ + if (!isfinite(flatValue) || flatValue <= 0.0 || \ + (flatMask && flatMask->data.U8[j + yOffset][i + xOffset])) { \ + if (inMask) { \ + inMask->data.PS_TYPE_MASK_DATA[j][i] |= badFlat; \ + } \ + inImage->data.TYPE[j][i] = SPECIAL; \ + } else { \ + inImage->data.TYPE[j][i] /= flatValue; \ + } \ + } \ + } \ + break; + + switch (inImage->type.type) { + FLAT_DIVISION_CASE(U8, 0); + FLAT_DIVISION_CASE(U16, 0); + FLAT_DIVISION_CASE(U32, 0); + FLAT_DIVISION_CASE(U64, 0); + FLAT_DIVISION_CASE(S8, 0); + FLAT_DIVISION_CASE(S16, 0); + FLAT_DIVISION_CASE(S32, 0); + FLAT_DIVISION_CASE(S64, 0); + FLAT_DIVISION_CASE(F32, NAN); + FLAT_DIVISION_CASE(F64, NAN); + default: + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Unsupported type for input image: %x\n", + inImage->type.type); + return false; + } + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now, used for reporting + psString timeString = psTimeToISO(time); // String with time + psFree(time); + psStringPrepend(&timeString, "Flat-field processing completed at "); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + timeString, ""); + psFree(timeString); + + + return true; +} Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFlatField.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFlatField.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFlatField.h (revision 22293) @@ -0,0 +1,30 @@ +/* @file pmFlatField.h + * @brief Apply flat field calibration + * + * @author Ross Harman, MHPCC + * @author Paul Price, IfA + * + * @version $Revision: 1.10 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-06-02 03:51:03 $ + * Copyright 2004-2006 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_FLAT_FIELD_H +#define PM_FLAT_FIELD_H + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +/// Apply flat field calibration to a readout +/// +/// This function applies the flat field calibration to the input readout. Support is available for different +/// image types, though the input and flat images must have the same type. The relative offsets between the +/// input and flat images is determined from the readout row0,col0 and the CELL.X0 and CELL.Y0 concepts. +/// Normalisation of the flat is left as the responsibility of the caller. Non-positive pixels in the flat +/// are masked, if there is a mask present in the input readout. +bool pmFlatField(pmReadout *in, ///< Readout with input image + const pmReadout *flat, ///< Readout with flat image + psMaskType badFlat ///< Mask value to give bad flat pixels + ); +/// @} +#endif Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFlatNormalize.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFlatNormalize.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFlatNormalize.c (revision 22293) @@ -0,0 +1,189 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmFlatNormalize.h" + +// I'm not sure that many many iterations are required, but rather suspect that the system converges within a +// few with absolutely no trouble (it *is* over-constrained). For this reason, I'm putting the maximum number +// of iterations and tolerance as preset values. +#define MAXITER 10 // Maximum number of iterations +#define TOLERANCE 1e-3 // Minimum tolerance for convergance + + +bool pmFlatNormalize(psVector **expFluxesPtr, psVector **chipGainsPtr, const psImage *bgMatrix) +{ + PS_ASSERT_PTR_NON_NULL(bgMatrix, false); + PS_ASSERT_IMAGE_NON_NULL(bgMatrix, false); + + int numExps = bgMatrix->numRows; // Number of exposures + int numChips = bgMatrix->numCols; // Number of chips with which each exposure is made + + psVector *expFluxes; // Dereferenced version of expFluxesPtr + if (expFluxesPtr) { + if (*expFluxesPtr) { + PS_ASSERT_VECTOR_TYPE(*expFluxesPtr, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_SIZE(*expFluxesPtr, (long)numExps, false); + } else { + *expFluxesPtr = psVectorAlloc(numExps, PS_TYPE_F32); + } + expFluxes = psMemIncrRefCounter(*expFluxesPtr); + } else { + expFluxes = psVectorAlloc(numExps, PS_TYPE_F32); + } + + psVector *chipGains; // Dereferenced version of chipGainsPtr + if (chipGainsPtr) { + if (*chipGainsPtr) { + PS_ASSERT_VECTOR_TYPE(*chipGainsPtr, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_SIZE(*chipGainsPtr, (long)numChips, false); + } else { + *chipGainsPtr = psVectorAlloc(numChips, PS_TYPE_F32); + psVectorInit(*chipGainsPtr, 1.0); + } + chipGains = psMemIncrRefCounter(*chipGainsPtr); + } else { + chipGains = psVectorAlloc(numChips, PS_TYPE_F32); + psVectorInit(chipGains, 1.0); + } + + // Take the logarithms + psImage *flux = psImageCopy(NULL, bgMatrix, PS_TYPE_F32); // Copy of the input flux levels matrix + psImage *fluxMask = psImageAlloc(numChips, numExps, PS_TYPE_U8); // Mask for bad measurements + psImageInit(fluxMask, 0); + psVector *gainMask = psVectorAlloc(numChips, PS_TYPE_U8); // Mask for bad gains + psVectorInit(gainMask, 0); + psVector *expMask = psVectorAlloc(numExps, PS_TYPE_U8); // Mask for bad exposures + psVectorInit(expMask, 0); + for (int i = 0; i < numChips; i++) { + // Note: the input gains are in e/ADU; we want to work with ADU/e (bg [ADU] = g [ADU/e] * f [e]) + // Hence the minus sign + if (isfinite(chipGains->data.F32[i]) && chipGains->data.F32[i] > 0) { + chipGains->data.F32[i] = -logf(chipGains->data.F32[i]); + } else { + chipGains->data.F32[i] = 0.0; // Take a wild guess, gain ~ 1 e/ADU + } + + for (int j = 0; j < numExps; j++) { + if (isfinite(flux->data.F32[j][i]) && flux->data.F32[j][i] > 0) { + flux->data.F32[j][i] = logf(flux->data.F32[j][i]); + } else { + // Blank out this measurement + fluxMask->data.U8[j][i] = 1; + flux->data.F32[j][i] = NAN; + } + } + } + + // Not really sure that we need to iterate, but here we go anyway... + + float diff = INFINITY; // Difference from previous iteration + psVector *oldExpFluxes = NULL; // The fluxes in the previous iteration + psVector *oldChipGains = NULL; // Chip gains in the previous iteration + for (int iter = 0; iter < MAXITER && diff > TOLERANCE; iter++) { + // Improve on the exposure fluxes + int numFluxes = 0; // Number of fluxes + for (int i = 0; i < numExps; i++) { + if (expMask->data.U8[i]) { + psTrace("psModules.detrend", 7, "Flux for exposure %d is masked.\n", i); + continue; + } + numFluxes++; + float sum = 0.0; // Sum of F_ij - G_j + int number = 0; // Number of chips contributing + for (int j = 0; j < numChips; j++) { + if (!gainMask->data.U8[j] && !fluxMask->data.U8[i][j]) { + sum += flux->data.F32[i][j] - chipGains->data.F32[j]; + number++; + } + } + if (number > 0) { + expFluxes->data.F32[i] = sum / (float)number; + } else { + expMask->data.U8[i] = 1; + expFluxes->data.F32[i] = NAN; + } + psTrace("psModules.detrend", 7, "Flux for exposure %d is %lf\n", i, expf(expFluxes->data.F32[i])); + } + + // Improve on the gains + float meanGain = 0.0; // Mean gain + int numGains = 0; // Number of gains + for (int i = 0; i < numChips; i++) { + if (gainMask->data.U8[i]) { + continue; + } + float sum = 0.0; // Sum of F_ji - S_j + int number = 0; // Numer of sources contributing + for (int j = 0; j < numExps; j++) { + if (!fluxMask->data.U8[j][i]) { + sum += flux->data.F32[j][i] - expFluxes->data.F32[j]; + number++; + } + } + if (number > 0) { + chipGains->data.F32[i] = sum / (float)number; + } else { + gainMask->data.U8[i] = 1; + chipGains->data.F32[i] = NAN; + } + psTrace("psModules.detrend", 7, "Gain for chip %d is %lf\n", i, expf(-chipGains->data.F32[i])); + meanGain += expf(chipGains->data.F32[i]); + numGains++; + } + + // Normalise the mean gain to unity, and measure the difference + meanGain /= (float)numGains; + meanGain = logf(meanGain); + if (iter > 0) { + diff = 0.0; + for (int i = 0; i < numChips; i++) { + if (gainMask->data.U8[i]) { + continue; + } + chipGains->data.F32[i] -= meanGain; + diff += abs((chipGains->data.F32[i] - oldChipGains->data.F32[i]) / chipGains->data.F32[i]); + } + for (int i = 0; i < numExps; i++) { + if (expMask->data.U8[i]) { + continue; + } + diff += abs((expFluxes->data.F32[i] - oldExpFluxes->data.F32[i]) / expFluxes->data.F32[i]); + } + } + + psTrace("psModules.detrend", 2, "Iteration %d: difference is %e\n", iter, diff); + + // Copy the new to the old + oldChipGains = psVectorCopy(oldChipGains, chipGains, PS_TYPE_F32); + oldExpFluxes = psVectorCopy(oldExpFluxes, expFluxes, PS_TYPE_F32); + } + psFree(flux); + psFree(fluxMask); + psFree(oldChipGains); + psFree(oldExpFluxes); + + // Un-log the vectors + for (int i = 0; i < numChips; i++) { + if (!gainMask->data.U8[i]) { + chipGains->data.F32[i] = expf(chipGains->data.F32[i]); + } + } + for (int i = 0; i < numExps; i++) { + if (!expMask->data.U8[i]) { + expFluxes->data.F32[i] = expf(expFluxes->data.F32[i]); + } + } + psFree(gainMask); + psFree(expMask); + + psFree(chipGains); + psFree(expFluxes); + + return (diff < TOLERANCE); // Did we converge? +} Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFlatNormalize.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFlatNormalize.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFlatNormalize.h (revision 22293) @@ -0,0 +1,31 @@ +/* @file pmFlatNormalize.h + * @brief Normalize flat-field measurements + * + * @author Paul Price, IfA + * + * @version $Revision: 1.7 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-03-30 21:12:56 $ + * Copyright 2004-2006 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_FLAT_NORMALIZE_H +#define PM_FLAT_NORMALIZE_H + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +/// Normalize flat-field measurements +/// +/// We have f_ij = g_i s_j where f_ij is the flux recorded for chip i and integration j, g_i is the gain for +/// the i-th chip, s_j is the flux of the source in the j-th integration. An initial guess for the chip gains +/// might be helpful, but is not necessary. The matrix of background measurements contains the background for +/// the flat fields used in the combination, as a function of exposure (rows) and chip (columns). The +/// exposure fluxes and chip gains are modified upon return with the solved values. Returns true if the +/// solution converged. +bool pmFlatNormalize(psVector **expFluxesPtr, ///< Flux in each exposure, or NULL; modified + psVector **chipGainsPtr, ///< Initial guess of the chip gains or NULL; modified + const psImage *bgMatrix ///< Background measurements: rows are exposures, cols are chips + ); + +/// @} +#endif Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFringeStats.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFringeStats.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFringeStats.c (revision 22293) @@ -0,0 +1,1071 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmFringeStats.h" + +// Future optimisations for speed: +// +// 1. Clipping --- don't re-do the matrix setup again, but carry matrix and vector around, subtract +// contributions from clipped data points. +// 2. Faster psImageStats (use memcpy?) + + + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// pmFringeRegions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static void fringeRegionsFree(pmFringeRegions *fringe) +{ + psFree(fringe->x); + psFree(fringe->y); + psFree(fringe->mask); + return; +} + +pmFringeRegions *pmFringeRegionsAlloc(int nPts, int dX, int dY, int nX, int nY) +{ + pmFringeRegions *fringe = psAlloc(sizeof(pmFringeRegions)); + (void)psMemSetDeallocator(fringe, (psFreeFunc)fringeRegionsFree); + + fringe->x = NULL; + fringe->y = NULL; + fringe->mask = NULL; + + fringe->nRequested = nPts; + fringe->nAccepted = 0; + + fringe->dX = dX; + fringe->dY = dY; + fringe->nX = nX; + fringe->nY = nY; + + return fringe; +} + +bool pmFringeRegionsCreatePoints(pmFringeRegions *fringe, const psImage *image, psRandom *random) +{ + PS_ASSERT_PTR_NON_NULL(fringe, false); + PS_ASSERT_PTR_NON_NULL(image, false); + PS_ASSERT_IMAGE_NON_EMPTY(image, false); + + double frnd; + // create fringe->nRequested + + psRandom *rng; + if (random) { + rng = psMemIncrRefCounter(random); + } else { + rng = psRandomAlloc(PS_RANDOM_TAUS, 0); + } + + fringe->x = psVectorRecycle(fringe->x, fringe->nRequested, PS_TYPE_F32); + fringe->y = psVectorRecycle(fringe->y, fringe->nRequested, PS_TYPE_F32); + fringe->mask = psVectorRecycle(fringe->mask, fringe->nRequested, PS_TYPE_U8); + fringe->x->n = fringe->y->n = fringe->mask->n = fringe->nRequested; + psVectorInit(fringe->mask, 0); + + int nX = image->numCols; + int nY = image->numRows; + + psF32 *xPt = fringe->x->data.F32; + psF32 *yPt = fringe->y->data.F32; + + int dX = fringe->dX; + int dY = fringe->dY; + + // generate random points located within image bounds + for (int i = 0; i < fringe->nRequested; i++) { + frnd = psRandomUniform(rng); + xPt[i] = (nX - 2*dX)* frnd + dX; + frnd = psRandomUniform(rng); + yPt[i] = (nY - 2*dY)* frnd + dY; + } + + psFree(rng); + + return true; +} + +bool pmFringeRegionsWriteFits(psFits *fits, psMetadata *header, + const pmFringeRegions *regions, const char *extname) +{ + // Make sure the input is well-behaved + PS_ASSERT_PTR_NON_NULL(fits, false); + PS_ASSERT_PTR_NON_NULL(regions, false); + psVector *x = regions->x; // The x positions + psVector *y = regions->y; // The y positions + psVector *mask = regions->mask; // The region mask + int numRows = regions->nRequested; // Number of rows in the table + PS_ASSERT_INT_POSITIVE(numRows, false); + PS_ASSERT_VECTOR_NON_NULL(x, false); + PS_ASSERT_VECTOR_NON_NULL(y, false); + PS_ASSERT_VECTOR_TYPE(x, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_TYPE(y, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_SIZE(x, (long)numRows, false); + PS_ASSERT_VECTOR_SIZE(y, (long)numRows, false); + if (mask) { + PS_ASSERT_VECTOR_NON_NULL(mask, false); + PS_ASSERT_VECTOR_TYPE(mask, PS_TYPE_U8, false); + PS_ASSERT_VECTOR_SIZE(mask, (long)numRows, false); + } + + // We need to write: + // Scalars: dX, dY, nX, nY + // Vectors: x, y, mask + + psMetadata *scalars = psMemIncrRefCounter(header); // Metadata to hold the scalars; will be the header + if (!scalars) { + scalars = psMetadataAlloc(); + } + psMetadataAddS32(scalars, PS_LIST_TAIL, "PSFRNGDX", PS_META_REPLACE, "Median box half-width", + regions->dX); + psMetadataAddS32(scalars, PS_LIST_TAIL, "PSFRNGDY", PS_META_REPLACE, "Median box half-height", + regions->dY); + psMetadataAddS32(scalars, PS_LIST_TAIL, "PSFRNGNX", PS_META_REPLACE, "Large-scale smoothing in x", + regions->nX); + psMetadataAddS32(scalars, PS_LIST_TAIL, "PSFRNGNY", PS_META_REPLACE, "Large-scale smoothing in y", + regions->nY); + + psArray *table = psArrayAlloc(numRows); // The table + // Translate the vectors into the required format for psFitsWriteTable() + for (long i = 0; i < numRows; i++) { + psMetadata *row = psMetadataAlloc(); + psMetadataAddF32(row, PS_LIST_TAIL, "x", PS_META_REPLACE, "Fringe position in x", x->data.F32[i]); + psMetadataAddF32(row, PS_LIST_TAIL, "y", PS_META_REPLACE, "Fringe position in y", y->data.F32[i]); + psU8 maskValue = 0; + if (mask && mask->data.U8[i]) { + maskValue = 0xff; + } + psMetadataAddU8(row, PS_LIST_TAIL, "mask", PS_META_REPLACE, "Mask", maskValue); + table->data[i] = row; + } + + bool success; // Success of operation + if (!(success = psFitsWriteTable(fits, scalars, table, extname))) { + psError(PS_ERR_IO, false, "Unable to write fringe data to extension %s\n", extname); + } + psFree(scalars); + psFree(table); + + return success; +} + +pmFringeRegions *pmFringeRegionsReadFits(psMetadata *header, const psFits *fits, const char *extname) +{ + PS_ASSERT_PTR_NON_NULL(fits, NULL); + + if (extname && strlen(extname) > 0) { + if (!psFitsMoveExtName(fits, extname)) { + psError(PS_ERR_IO, false, "Unable to move to extension %s\n", extname); + return NULL; + } + } else if (!psFitsMoveExtNum(fits, 0, false)) { + psError(PS_ERR_IO, false, "Unable to move to PHU\n"); + return NULL; + } + + psMetadata *headerCopy = psMemIncrRefCounter(header); // Copy of the header, or NULL + + headerCopy = psFitsReadHeader(headerCopy, fits); // The FITS header + if (!headerCopy) { + psError(PS_ERR_IO, false, "Unable to read header for extension %s\n", extname); + psFree(header); + return NULL; + } + + // Read the scalars from the header + #define READ_SCALAR(SCALAR, NAME) \ + int SCALAR = psMetadataLookupS32(&mdok, headerCopy, NAME); \ + if (!mdok || SCALAR <= 0) { \ + psError(PS_ERR_IO, true, "Unable to find " NAME " in header of extension %s.\n", extname); \ + psFree(headerCopy); \ + return NULL; \ + } + + // Need to retrieve the scalars: dX, dY, nX, nY + bool mdok = true; // Status of MD lookup + READ_SCALAR(dX, "PSFRNGDX"); + READ_SCALAR(dY, "PSFRNGDY"); + READ_SCALAR(nX, "PSFRNGNX"); + READ_SCALAR(nY, "PSFRNGNY"); + psFree(headerCopy); + + // Now the vectors: x, y, mask + psArray *table = psFitsReadTable(fits); // The table + long numRows = table->n; // Number of rows + + pmFringeRegions *regions = pmFringeRegionsAlloc(numRows, dX, dY, nX, nY); // The fringe regions + psVector *x = psVectorAlloc(numRows, PS_TYPE_F32); // x position + psVector *y = psVectorAlloc(numRows, PS_TYPE_F32); // y position + psVector *mask = psVectorAlloc(numRows, PS_TYPE_U8); // mask + regions->x = x; + regions->y = y; + regions->mask = mask; + + #define READ_REGIONS_ROW(VECTOR, TYPE, NAME, DESCRIPTION) \ + VECTOR->data.TYPE[i] = psMetadataLookup##TYPE(&mdok, row, NAME); \ + if (!mdok) { \ + psError(PS_ERR_IO, true, "Unable to find " #DESCRIPTION " .\n"); \ + psFree(table); \ + psFree(regions); \ + return NULL; \ + } + + // Translate the table into vectors + for (long i = 0; i < numRows; i++) { + psMetadata *row = table->data[i]; // Table row + READ_REGIONS_ROW(x, F32, "x", "x position"); + READ_REGIONS_ROW(y, F32, "y", "y position"); + READ_REGIONS_ROW(mask, U8, "mask", "mask"); + } + psFree(table); + + return regions; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// pmFringeStats +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static void fringeStatsFree(pmFringeStats *stats) +{ + psFree(stats->regions); + psFree(stats->f); + psFree(stats->df); +} + +pmFringeStats *pmFringeStatsAlloc(pmFringeRegions *regions) +{ + PS_ASSERT_PTR_NON_NULL(regions, false); + + pmFringeStats *stats = psAlloc(sizeof(pmFringeStats)); + (void)psMemSetDeallocator(stats, (psFreeFunc)fringeStatsFree); + + int numRegions = regions->nRequested; // Number of regions + stats->regions = psMemIncrRefCounter(regions); + stats->f = psVectorAlloc(numRegions, PS_TYPE_F32); + stats->df = psVectorAlloc(numRegions, PS_TYPE_F32); + + return stats; +} + +pmFringeStats *pmFringeStatsMeasure(pmFringeRegions *fringe, const pmReadout *readout, psMaskType maskVal) +{ + PS_ASSERT_PTR_NON_NULL(fringe, NULL); + PS_ASSERT_PTR_NON_NULL(readout, NULL); + PS_ASSERT_PTR_NON_NULL(readout->image, NULL); + PS_ASSERT_IMAGE_NON_EMPTY(readout->image, NULL); + + if (!fringe->x || !fringe->y) { + // create the fringe vectors for this image + pmFringeRegionsCreatePoints(fringe, readout->image, NULL); + } + + PS_ASSERT_PTR_NON_NULL(fringe->x, false); + PS_ASSERT_PTR_NON_NULL(fringe->y, false); + + pmFringeStats *measurements = pmFringeStatsAlloc(fringe); + + psF32 *xPt = fringe->x->data.F32; + psF32 *yPt = fringe->y->data.F32; + psF32 *fPt = measurements->f->data.F32; + psF32 *dfPt = measurements->df->data.F32; + + int dX = fringe->dX; + int dY = fringe->dY; + + psImage *image = readout->image; + psImage *mask = readout->mask; + + psStats *median = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN); // Median statistics only + psStats *medianSd = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV); // Median and SD + + // Measure the sky in each smoothing box + psImage *sky = psImageAlloc(fringe->nX, fringe->nY, PS_TYPE_F32); + for (int i = 0; i < fringe->nY; i++) { + int y0 = image->row0 + (float)i * (float)image->numRows / (float)fringe->nY; + int y1 = image->row0 + (float)(i + 1) * (float)image->numRows / (float)fringe->nY; + for (int j = 0; j < fringe->nX; j++) { + int x0 = image->col0 + (float)j * (float)image->numCols / (float)fringe->nX; + int x1 = image->col0 + (float)(j + 1) * (float)image->numCols / (float)fringe->nX; + psRegion region = psRegionSet(x0, x1, y0, y1); + psImage *subImage = psImageSubset(image, region); // Subimage of the sky region + psImage *subMask = NULL; + if (mask) { + subMask = psImageSubset(mask, region); // Subimage of the sky region + } + psImageStats(median, subImage, subMask, maskVal); + sky->data.F32[i][j] = median->sampleMedian; + psFree(subImage); + psFree(subMask); + } + } + + for (int i = 0; i < fringe->x->n; i++) { + psRegion region = psRegionSet(image->col0 + xPt[i] - dX, + image->col0 + xPt[i] + dX + 1, + image->row0 + yPt[i] - dY, + image->row0 + yPt[i] + dY + 1); + psImage *subImage = psImageSubset(image, region); + psImage *subMask = NULL; + if (mask) { + subMask = psImageSubset(mask, region); + } + psImageStats(medianSd, subImage, subMask, maskVal); + psFree(subImage); + psFree(subMask); + + int xSky = xPt[i] / (float)image->numCols * (float)sky->numCols; + int ySky = yPt[i] / (float)image->numRows * (float)sky->numRows; + + fPt[i] = medianSd->sampleMedian - sky->data.F32[ySky][xSky]; + dfPt[i] = 1.0 / medianSd->sampleStdev; + + psTrace("psModules.detrend", 7, "[%d:%d,%d:%d]: %f %f\n", (int)region.x0, (int)region.x1, + (int)region.y0, (int)region.y1, fPt[i], dfPt[i]); + } + psFree(sky); + psFree(median); + psFree(medianSd); + + return measurements; +} + +bool pmFringeStatsWriteFits(psFits *fits, + psMetadata *header, + const pmFringeStats *fringe, + const char *extname + ) +{ + // Make sure the input is well-behaved + PS_ASSERT_PTR_NON_NULL(fits, false); + PS_ASSERT_PTR_NON_NULL(fringe, false); + pmFringeRegions *regions = fringe->regions; // The fringe regions + PS_ASSERT_PTR_NON_NULL(regions, false); + int numRows = regions->nRequested; // Number of rows in the table + PS_ASSERT_INT_POSITIVE(numRows, false); + psVector *f = fringe->f; // The fringe measurements + psVector *df = fringe->df; // The fringe standard deviatiations + PS_ASSERT_VECTOR_NON_NULL(f, false); + PS_ASSERT_VECTOR_NON_NULL(df, false); + PS_ASSERT_VECTOR_TYPE(f, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_TYPE(df, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_SIZE(f, (long)numRows, false); + PS_ASSERT_VECTOR_SIZE(df, (long)numRows, false); + + // We need to write: + // Vectors: f, df + psArray *table = psArrayAlloc(numRows); // The table + // Translate the vectors into the required format for psFitsWriteTable() + for (long i = 0; i < numRows; i++) { + psMetadata *row = psMetadataAlloc(); + psMetadataAddF32(row, PS_LIST_TAIL, "f", PS_META_REPLACE, "Fringe measurement", f->data.F32[i]); + psMetadataAddF32(row, PS_LIST_TAIL, "df", PS_META_REPLACE, "Fringe stdev", df->data.F32[i]); + table->data[i] = row; + } + + if (!psFitsWriteTable(fits, header, table, extname)) { + psError(PS_ERR_IO, false, "Unable to write fringe data to extension %s\n", extname); + psFree(table); + return false; + } + + psFree(table); + return true; +} + +pmFringeStats *pmFringeStatsReadFits(psMetadata *header, const psFits *fits, const char *extname, + pmFringeRegions *regions) +{ + PS_ASSERT_PTR_NON_NULL(fits, NULL); + PS_ASSERT_PTR_NON_NULL(regions, NULL); + PS_ASSERT_INT_POSITIVE(regions->nRequested, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(regions->x, regions->y, NULL); + PS_ASSERT_VECTOR_SIZE(regions->x, (long)regions->nRequested, NULL); + + if (extname && strlen(extname) > 0) { + if (!psFitsMoveExtName(fits, extname)) { + psError(PS_ERR_IO, false, "Unable to move to extension %s\n", extname); + return NULL; + } + } else if (!psFitsMoveExtNum(fits, 0, false)) { + psError(PS_ERR_IO, false, "Unable to move to PHU\n"); + return NULL; + } + + psMetadata *headerCopy = psMemIncrRefCounter(header); // Copy of the header, or NULL + + headerCopy = psFitsReadHeader(headerCopy, fits); // The FITS header + if (!headerCopy) { + psError(PS_ERR_IO, false, "Unable to read header for extension %s\n", extname); + psFree(headerCopy); + return NULL; + } + psFree(headerCopy); + + // Now the vectors: f, df + psArray *table = psFitsReadTable(fits); // The table + long numRows = table->n; // Number of rows + + pmFringeStats *fringes = pmFringeStatsAlloc(regions); // The fringe measurements + psVector *f = fringes->f; // fringe measurement + psVector *df = fringes->df; // fringe stdev + + #define READ_STATS_ROW(VECTOR, TYPE, NAME, DESCRIPTION) \ + VECTOR->data.TYPE[i] = psMetadataLookup##TYPE(&mdok, row, NAME); \ + if (!mdok) { \ + psError(PS_ERR_IO, true, "Unable to find " #DESCRIPTION " .\n"); \ + psFree(table); \ + psFree(fringes); \ + return NULL; \ + } + + // Translate the table into vectors + bool mdok; // Status of MD lookup + for (long i = 0; i < numRows; i++) { + psMetadata *row = table->data[i]; // Table row + READ_STATS_ROW(f, F32, "f", "fringe measurement"); + READ_STATS_ROW(df, F32, "df", "fringe standard deviation"); + } + psFree(table); + + return fringes; +} + + +pmFringeStats *pmFringeStatsConcatenate(const psArray *fringes, const psVector *x0, const psVector *y0) +{ + PS_ASSERT_PTR_NON_NULL(fringes, NULL); + PS_ASSERT_PTR_NON_NULL(fringes->data, NULL); + PS_ASSERT_INT_POSITIVE(fringes->n, NULL); + if (x0 && y0) { + PS_ASSERT_VECTOR_NON_NULL(x0, NULL); + PS_ASSERT_VECTOR_NON_NULL(y0, NULL); + PS_ASSERT_VECTOR_TYPE(x0, PS_TYPE_S32, NULL); + PS_ASSERT_VECTOR_TYPE(y0, PS_TYPE_S32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(x0, y0, NULL); + PS_ASSERT_VECTOR_SIZE(x0, fringes->n, NULL); + PS_ASSERT_VECTOR_SIZE(y0, fringes->n, NULL); + } + + // Get the measurement parameters, and check they are consistent + int numPoints = 0; // Number of fringe points + int dX = 0, dY = 0; // Half-width and -height of fringe boxes + int nX = 0, nY = 0; // Smoothing scales + for (long i = 0; i < fringes->n; i++) { + pmFringeStats *fringe = fringes->data[i]; // The fringe of interest + pmFringeRegions *regions = fringe->regions; // The fringe regions + if (numPoints == 0) { + dX = regions->dX; + dY = regions->dY; + nX = regions->nX; + nY = regions->nY; + } else if (regions->dX != dX || regions->dY != dY || regions->nX != nX || regions->nY != nY) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Fringe %ld has different parameters (%d,%d,%d,%d) " + "from the first (%d,%d,%d,%d).\n", i, + regions->dX, regions->dY, regions->nX, regions->nY, dX, dY, nX, nY); + return NULL; + } + int num = regions->nRequested; // Number of fringe points + if (regions->x->n != num || regions->y->n != num || regions->mask->n != num || + fringe->f->n != num || fringe->df->n != num) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Length of region (%ld,%ld,%ld) and fringe vectors " + "do not match (%ld,%ld) with the official value (%d).\n", regions->x->n, regions->y->n, + regions->mask->n, fringe->f->n, fringe->df->n, num); + return NULL; + } + numPoints += regions->nRequested; + } + + pmFringeRegions *newRegions = pmFringeRegionsAlloc(numPoints, dX, dY, nX, nY); // The new list of regions + newRegions->x = psVectorAlloc(numPoints, PS_TYPE_F32); + newRegions->y = psVectorAlloc(numPoints, PS_TYPE_F32); + newRegions->mask = psVectorAlloc(numPoints, PS_TYPE_U8); + pmFringeStats *newStats = pmFringeStatsAlloc(newRegions); // The new list of statistics + + long offset = 0; // Offset from start of the list + for (long i = 0; i < fringes->n; i++) { + pmFringeStats *fringe = fringes->data[i]; // The fringe of interest + pmFringeRegions *regions = fringe->regions; // The fringe regions + // Copy the data over + memcpy(&newRegions->x->data.F32[offset], regions->x->data.F32, regions->x->n * sizeof(psF32)); + memcpy(&newRegions->y->data.F32[offset], regions->y->data.F32, regions->y->n * sizeof(psF32)); + memcpy(&newRegions->mask->data.U8[offset], regions->mask->data.U8, regions->mask->n * sizeof(psU8)); + memcpy(&newStats->f->data.F32[offset], fringe->f->data.F32, fringe->f->n * sizeof(psF32)); + memcpy(&newStats->df->data.F32[offset], fringe->df->data.F32, fringe->df->n * sizeof(psF32)); + if (x0 && y0) { + for (long j = offset; j < offset + regions->x->n; j++) { + newRegions->x->data.F32[j] += x0->data.S32[i]; + newRegions->y->data.F32[j] += y0->data.S32[i]; + } + } + offset += regions->nRequested; + } + + psFree(newRegions); // Drop reference + return newStats; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// pmFringeIO +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +bool pmFringesFormat(pmCell *cell, psMetadata *header, const psArray *fringes) +{ + PS_ASSERT_PTR_NON_NULL(cell, false); + PS_ASSERT_ARRAY_NON_NULL(fringes, false); + + // Check the regions are all identical + pmFringeRegions *regions = ((pmFringeStats*)fringes->data[0])->regions; // First region + for (int i = 1; i < fringes->n; i++) { + pmFringeStats *stats = fringes->data[i]; + if (stats->regions != regions) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Regions for fringe statistics are not identical.\n"); + return false; + } + } + + // Ensure the region is legit + psVector *x = regions->x; // The x positions + psVector *y = regions->y; // The y positions + psVector *mask = regions->mask; // The region mask + int numRows = regions->nRequested; // Number of rows in the table + PS_ASSERT_INT_POSITIVE(numRows, false); + PS_ASSERT_VECTOR_NON_NULL(x, false); + PS_ASSERT_VECTOR_NON_NULL(y, false); + PS_ASSERT_VECTOR_TYPE(x, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_TYPE(y, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_SIZE(x, (long)numRows, false); + PS_ASSERT_VECTOR_SIZE(y, (long)numRows, false); + if (mask) { + PS_ASSERT_VECTOR_NON_NULL(mask, false); + PS_ASSERT_VECTOR_TYPE(mask, PS_TYPE_U8, false); + PS_ASSERT_VECTOR_SIZE(mask, (long)numRows, false); + } + + // We need to write: + // Scalars: dX, dY, nX, nY + // Vectors: x, y, mask, f, df + + psMetadata *scalars = psMemIncrRefCounter(header); // Metadata to hold the scalars; will be the header + if (!scalars) { + scalars = psMetadataAlloc(); + } + psMetadataAddS32(scalars, PS_LIST_TAIL, "PSFRNGDX", PS_META_REPLACE, "Median box half-width", + regions->dX); + psMetadataAddS32(scalars, PS_LIST_TAIL, "PSFRNGDY", PS_META_REPLACE, "Median box half-height", + regions->dY); + psMetadataAddS32(scalars, PS_LIST_TAIL, "PSFRNGNX", PS_META_REPLACE, "Large-scale smoothing in x", + regions->nX); + psMetadataAddS32(scalars, PS_LIST_TAIL, "PSFRNGNY", PS_META_REPLACE, "Large-scale smoothing in y", + regions->nY); + psMetadataAdd(cell->analysis, PS_LIST_TAIL, "FRINGE.HEADER", PS_DATA_METADATA, + "Header for fringe data", scalars); + psFree(scalars); + + + psArray *table = psArrayAlloc(numRows); // The table + // Translate the vectors into the required format for psFitsWriteTable() + for (long i = 0; i < numRows; i++) { + psMetadata *row = psMetadataAlloc(); + psMetadataAddF32(row, PS_LIST_TAIL, "x", PS_META_REPLACE, "Fringe position in x", x->data.F32[i]); + psMetadataAddF32(row, PS_LIST_TAIL, "y", PS_META_REPLACE, "Fringe position in y", y->data.F32[i]); + psU8 maskValue = 0; // Mask value + if (mask && mask->data.U8[i]) { + maskValue = 0xff; + } + + psVector *f = psVectorAlloc(fringes->n, PS_TYPE_F32); // Measurements for each fringe component + psVector *df = psVectorAlloc(fringes->n, PS_TYPE_F32); // Errors in measurements + for (long j = 0; j < fringes->n; j++) { + pmFringeStats *stats = fringes->data[j]; // Fringe statistics of interest + f->data.F32[j] = stats->f->data.F32[i]; + df->data.F32[j] = stats->df->data.F32[i]; + if (!isfinite(f->data.F32[j]) || !isfinite(df->data.F32[j])) { + maskValue = 0xff; + } + } + psMetadataAdd(row, PS_LIST_TAIL, "f", PS_DATA_VECTOR | PS_META_REPLACE, "Fringe measurements", f); + psMetadataAdd(row, PS_LIST_TAIL, "df", PS_DATA_VECTOR | PS_META_REPLACE, "Fringe errors", df); + // Drop references + psFree(f); + psFree(df); + + psMetadataAddU8(row, PS_LIST_TAIL, "mask", PS_META_REPLACE, "Mask", maskValue); + table->data[i] = row; + } + + psMetadataAdd(cell->analysis, PS_LIST_TAIL, "FRINGE", PS_DATA_ARRAY, "Fringe data", table); + + return true; +} + + +psArray *pmFringesParse(pmCell *cell) +{ + PS_ASSERT_PTR_NON_NULL(cell, NULL); + + bool mdok; // Status of MD lookup + psMetadata *header = psMetadataLookupMetadata(&mdok, cell->analysis, "FRINGE.HEADER"); // Header + if (!mdok || !header) { + psError(PS_ERR_UNKNOWN, false, "Unable to find header for fringe data.\n"); + return NULL; + } + + psArray *table = psMetadataLookupPtr(NULL, cell->analysis, "FRINGE"); // FITS table + if (!table) { + psError(PS_ERR_UNKNOWN, false, "Unable to find table for fringe data.\n"); + return NULL; + } + + + // Read the scalars from the header + #define READ_FRINGES_SCALAR(SCALAR, NAME) \ + int SCALAR = psMetadataLookupS32(&mdok, header, NAME); \ + if (!mdok || SCALAR <= 0) { \ + psError(PS_ERR_IO, true, "Unable to find " NAME " in fringe header.\n"); \ + return NULL; \ + } + + // Need to retrieve the scalars: dX, dY, nX, nY + READ_FRINGES_SCALAR(dX, "PSFRNGDX"); + READ_FRINGES_SCALAR(dY, "PSFRNGDY"); + READ_FRINGES_SCALAR(nX, "PSFRNGNX"); + READ_FRINGES_SCALAR(nY, "PSFRNGNY"); + + // Now the vectors: x, y, mask, f, df + long numRows = table->n; // Number of rows + pmFringeRegions *regions = pmFringeRegionsAlloc(numRows, dX, dY, nX, nY); // The fringe regions + psVector *x = psVectorAlloc(numRows, PS_TYPE_F32); // x position + psVector *y = psVectorAlloc(numRows, PS_TYPE_F32); // y position + psVector *mask = psVectorAlloc(numRows, PS_TYPE_U8); // mask + regions->x = x; + regions->y = y; + regions->mask = mask; + psArray *f = psArrayAlloc(numRows); // Array of fringe measurements + psArray *df = psArrayAlloc(numRows);// Array of errors + psArray *fringes = NULL; // Array of fringes, to return + + #define READ_FRINGES_VECTOR_ROW(VECTOR, TYPE, NAME, DESCRIPTION) \ + { \ + VECTOR->data.TYPE[i] = psMetadataLookup##TYPE(&mdok, row, NAME); \ + if (!mdok) { \ + psError(PS_ERR_IO, true, "Unable to find " #DESCRIPTION " for row %ld.\n", i); \ + goto READ_FRINGES_DONE; \ + } \ + } + + // Some values may be either a vector or a value --- need to check + #define READ_FRINGES_ARRAY_ROW(ARRAY, TYPE, NAME, DESCRIPTION) \ + { \ + psMetadataItem *item = psMetadataLookup(row, NAME); \ + if (!item) { \ + psError(PS_ERR_IO, true, "Unable to find " #DESCRIPTION " for row %ld.\n", i); \ + goto READ_FRINGES_DONE; \ + } \ + if (item->type == PS_DATA_VECTOR) { \ + ARRAY->data[i] = psMemIncrRefCounter(item->data.V); \ + } else if (item->type == PS_TYPE_##TYPE) { \ + psVector *vector = psVectorAlloc(1, PS_TYPE_##TYPE); \ + vector->data.TYPE[0] = item->data.TYPE; \ + ARRAY->data[i] = vector; \ + } else { \ + psError(PS_ERR_IO, true, "Found " #DESCRIPTION " for row %ld, but it's of an " \ + "unsupported type (%x).\n", i, item->type); \ + goto READ_FRINGES_DONE; \ + } \ + } + + // Translate the table into vectors + for (long i = 0; i < numRows; i++) { + psMetadata *row = table->data[i]; // Table row + READ_FRINGES_VECTOR_ROW(x, F32, "x", "x position"); + READ_FRINGES_VECTOR_ROW(y, F32, "y", "y position"); + READ_FRINGES_VECTOR_ROW(mask, U8, "mask", "mask"); + READ_FRINGES_ARRAY_ROW(f, F32, "f", "fringe measurement"); + READ_FRINGES_ARRAY_ROW(df, F32, "df", "fringe error"); + } + + // Get f,df into pmFringeStats + long numFringes = ((psVector*)(f->data[0]))->n; // Number of fringe components + fringes = psArrayAlloc(numFringes); + for (int j = 0; j < numFringes; j++) { + fringes->data[j] = pmFringeStatsAlloc(regions); + } + + for (long i = 0; i < numRows; i++) { + psVector *measurements = f->data[i]; // Vector of measurements + psVector *errors = df->data[i]; // Vector of errors + for (int j = 0; j < numFringes; j++) { + pmFringeStats *fringe = fringes->data[j]; + fringe->f->data.F32[i] = measurements->data.F32[j]; + fringe->df->data.F32[i] = errors->data.F32[j]; + } + } + +READ_FRINGES_DONE: + psFree(regions); + psFree(f); + psFree(df); + + return fringes; +} + + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// pmFringeScale +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static void fringeScaleFree(pmFringeScale *scale) +{ + psFree(scale->coeff); + psFree(scale->coeffErr); + return; +} + +pmFringeScale *pmFringeScaleAlloc(int nFringeFrames) +{ + pmFringeScale *scale = psAlloc(sizeof(pmFringeScale)); + (void)psMemSetDeallocator(scale, (psFreeFunc)fringeScaleFree); + + scale->nFringeFrames = nFringeFrames; + scale->coeff = psVectorAlloc(nFringeFrames + 1, PS_TYPE_F32); + scale->coeffErr = psVectorAlloc(nFringeFrames + 1, PS_TYPE_F32); + + return scale; +} + +// Determine the fringe scales through solving the least-squares problem +static bool scaleMeasure(pmFringeScale *scale, // Scale to return + pmFringeStats *science, // The fringe measurements for the science image + psArray *fringes // Array of fringe measurements for the templates + ) +{ + assert(scale); + assert(science); + assert(fringes); + assert(scale->nFringeFrames == fringes->n); + + psVector *mask = science->regions->mask; // The region mask + + int numCoeffs = fringes->n + 1; // Number of coefficients: scales for the templates plus a background + int numPoints = science->regions->nRequested; // Number of points (i.e., fringe measurements) + + psImage *A = psImageAlloc(numCoeffs, numCoeffs, PS_TYPE_F64); // The least-squares matrix + psVector *B = psVectorAlloc(numCoeffs, PS_TYPE_F64); // The least-squares vector + + // Generate the least-squares matrix and vector + for (int i = 0; i < numCoeffs; i++) { + psVector *fringe1 = NULL; // A fringe measurement + if (i != 0) { + pmFringeStats *fringe = fringes->data[i - 1]; + fringe1 = fringe->f; + } + + // Fill in the upper part of the matrix + for (int j = i; j < numCoeffs; j++) { + psVector *fringe2 = NULL; // Another fringe measurement + if (j != 0) { + pmFringeStats *fringe = fringes->data[j - 1]; + fringe2 = fringe->f; + } + + double matrix = 0.0; // The matrix sum + for (int k = 0; k < numPoints; k++) { + if (!mask->data.U8[k]) { + psF32 f1 = (fringe1) ? fringe1->data.F32[k] : 1.0; // Contribution from i fringe + psF32 f2 = (fringe2) ? fringe2->data.F32[k] : 1.0; // Contribution from j fringe + psF32 dsInv = science->df->data.F32[k]; // 1 / sigma + matrix += f1 * f2 * dsInv * dsInv; + } + } + A->data.F64[i][j] = matrix; + } + + // Use symmetry to fill in the lower part of the matrix + for (int j = 0; j < i; j++) { + A->data.F64[i][j] = A->data.F64[j][i]; + } + + double vector = 0.0; // The vector sum + for (int k = 0; k < numPoints; k++) { + if (!mask->data.U8[k]) { + psF32 f1 = (fringe1) ? fringe1->data.F32[k] : 1.0; // Contribution from fringe 1 + psF32 s = science->f->data.F32[k]; // Contribution from science measurement + psF32 dsInv = science->df->data.F32[k]; // 1 / sigma + vector += f1 * s * dsInv * dsInv; + } + } + B->data.F64[i] = vector; + } + + if (psTraceGetLevel("psModules.detrend") >= 5) { + printf("From %d points:\n", numPoints); + for (int i = 0; i < numCoeffs; i++) { + for (int j = 0; j < numCoeffs; j++) { + printf("%.2e ", A->data.F64[i][j]); + } + printf("\n"); + } + } + + // Solve the least-squares equation + if (! psMatrixGJSolve(A, B)) { + psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations. Returning NULL.\n"); + return false; + } + + // Copy the results over + for (int i = 0; i < numCoeffs; i++) { + scale->coeff->data.F32[i] = B->data.F64[i]; + scale->coeffErr->data.F32[i] = sqrt(A->data.F64[i][i]); + } + + psFree(A); + psFree(B); + + return true; +} + +// Measure the fringe differences for each region +static bool fringeScaleDiffs(psVector *diff, // Vector of differences + pmFringeStats *science, // Science fringe measurements + psArray *fringes, // Template fringe measurements + pmFringeScale *scale // Fringe scales + ) +{ + assert(diff); + assert(diff->type.type == PS_TYPE_F32); + assert(science); + assert(fringes); + assert(scale); + assert(diff->n == science->regions->nRequested); + assert(fringes->n == scale->nFringeFrames); + + psVector *mask = science->regions->mask; // The region mask + + for (int i = 0; i < diff->n; i++) { + if (!mask->data.U8[i]) { + float difference = science->f->data.F32[i] - scale->coeff->data.F32[0]; + for (int j = 0; j < fringes->n; j++) { + pmFringeStats *fringe = fringes->data[j]; // The fringe of interest + difference -= scale->coeff->data.F32[j + 1] * fringe->f->data.F32[i]; + } + diff->data.F32[i] = difference * difference * science->df->data.F32[i] * science->df->data.F32[i]; + } + } + + return true; +} + +// Clip regions based on the differences; return the number masked +static int clipRegions(psVector *diffs, // Differences + psVector *mask, // Region mask + float rej // Rejection limit in standard deviations + ) +{ + assert(diffs); + assert(diffs->type.type == PS_TYPE_F32); + assert(mask); + assert(mask->type.type == PS_TYPE_U8); + assert(diffs->n == mask->n); + + psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_QUARTILE); // Statistics + psVectorStats(stats, diffs, NULL, mask, 1); + float middle = stats->sampleMedian; // The middle of the distribution + float thresh = rej * 0.74 * (stats->sampleUQ - stats->sampleLQ); // The rejection threshold + psFree(stats); + + int numClipped = 0; // Number clipped + for (int i = 0; i < diffs->n; i++) { + psTrace("psModules.detrend", 10, "Region %d (%d): %f\n", i, mask->data.U8[i], diffs->data.F32[i]); + if (!mask->data.U8[i] && fabs(diffs->data.F32[i]) > middle + thresh) { + psTrace("psModules.detrend", 5, "Masking %d: %f\n", i, diffs->data.F32[i]); + mask->data.U8[i] = 1; + numClipped++; + } + } + + return numClipped; +} + + +// XXX include the fringe error (fringe->df) in the fit? +pmFringeScale *pmFringeScaleMeasure(pmFringeStats *science, psArray *fringes, float rej, + unsigned int nIter, float keepFrac) +{ + PS_ASSERT_PTR_NON_NULL(science, NULL); + PS_ASSERT_PTR_NON_NULL(fringes, NULL); + PS_ASSERT_INT_POSITIVE(fringes->n, NULL); + PS_ASSERT_INT_POSITIVE(nIter, NULL); + + pmFringeRegions *regions = science->regions; // The fringe regions + int numRegions = regions->nRequested; // Number of regions + + // Ensure we are dealing with the SAME fringe points for all the inputs. + // Otherwise, we're going to get crazy results. + for (long i = 0; i < numRegions; i++) { + float xScience = regions->x->data.F32[i]; // The x position for the science image + float yScience = regions->y->data.F32[i]; // The y position for the science image + for (long j = 0; j < fringes->n; j++) { + pmFringeStats *fringe = fringes->data[j]; // The fringe statistics from a fringe image + pmFringeRegions *fringeRegions = fringe->regions; // The fringe regions for that fringe image + if (fringeRegions->x->data.F32[i] != xScience || + fringeRegions->y->data.F32[i] != yScience) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Science and fringe measurement regions " + "don't match.\n"); + return NULL; + } + } + } + + // Set up the mask + if (!regions->mask) { + regions->mask = psVectorAlloc(numRegions, PS_TYPE_U8); + psVectorInit(regions->mask, 0); + } + psVector *mask = regions->mask; // The region mask + psStats *median = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN); // Median statistics + unsigned int numClipped = 0; // Total number clipped + psVector *diff = psVectorAlloc(numRegions, PS_TYPE_F32); // The differences between obs. and pred. + + pmFringeScale *scale = pmFringeScaleAlloc(fringes->n); // The fringe scales + + // Get rid of bad data points + for (int i = 0; i < fringes->n; i++) { + pmFringeStats *fringe = fringes->data[i]; // The fringe of interest + for (int j = 0; j < numRegions; j++) { + if (!isfinite(fringe->f->data.F32[j])) { + mask->data.U8[j] = 1; + psTrace("psModules.detrend", 9, "Masking region %d because not finite in fringe %d.\n", j, i); + } + } + } + +# if (0) + // Write fringe data to file for a test + FILE *f = fopen ("fringe.dat", "w"); + for (int j = 0; j < numRegions; j++) { + if (mask->data.U8[j]) continue; + fprintf (f, "%d %f %f ", j, science->f->data.F32[j], science->df->data.F32[j]); + for (int i = 0; i < fringes->n; i++) { + pmFringeStats *fringe = fringes->data[i]; // The fringe of interest + fprintf (f, "%f ", fringe->f->data.F32[j]); + } + fprintf (f, "\n"); + } + fclose (f); +# endif + + // Get rid of the extreme outliers by assuming most of the points are somewhat clustered + psVectorStats(median, science->f, NULL, NULL, 0); + scale->coeff->data.F32[0] = median->sampleMedian; + for (int i = 0; i < fringes->n; i++) { + pmFringeStats *fringe = fringes->data[i]; // The fringe of interest + psVectorStats(median, fringe->f, NULL, NULL, 0); + scale->coeff->data.F32[0] -= median->sampleMedian; + scale->coeff->data.F32[i] = 0.0; + } + psFree(median); + fringeScaleDiffs(diff, science, fringes, scale); + numClipped = clipRegions(diff, mask, 3.0*rej); + psTrace("psModules.detrend", 4, "%d regions clipped in initial pass.\n", numClipped); + + unsigned int iter = 0; // Iteration number + unsigned int iterClip = 0; // Number clipped in this iteration + do { + iter++; + scaleMeasure(scale, science, fringes); // The scales + psTrace("psModules.detrend", 1, "Fringe scales after iteration %d:\n", iter); + psTrace("psModules.detrend", 1, "Background: %f %f\n", scale->coeff->data.F32[0], + scale->coeffErr->data.F32[0]); + for (int i = 0; i < scale->nFringeFrames; i++) { + psTrace("psModules.detrend", 1, "%d: %f %f\n", i, scale->coeff->data.F32[i + 1], + scale->coeffErr->data.F32[i + 1]); + } + + fringeScaleDiffs(diff, science, fringes, scale); + iterClip = clipRegions(diff, mask, rej); // Number clipped + numClipped += iterClip; + psTrace("psModules.detrend", 9, "Clipped: %d\tFrac: %f\n", iterClip, + (float)numClipped/(float)numRegions); + } while (iterClip > 0 && iter < nIter && (float)numClipped/(float)numRegions <= 1.0 - keepFrac); + psFree(diff); + + // A final iteration with the last clipping + scaleMeasure(scale, science, fringes); + + return scale; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Fringe correction +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// XXX note that this modifies the input fringe images +psImage *pmFringeCorrect(pmReadout *readout, pmFringeRegions *fringes, psArray *fringeImages, + psArray *fringeStats, psMaskType maskVal, float rej, + unsigned int nIter, float keepFrac) +{ + PS_ASSERT_PTR_NON_NULL(readout, NULL); + PS_ASSERT_PTR_NON_NULL(readout->image, NULL); + PS_ASSERT_IMAGE_NON_EMPTY(readout->image, NULL); + PS_ASSERT_PTR_NON_NULL(fringes, NULL); + PS_ASSERT_PTR_NON_NULL(fringeImages, NULL); + PS_ASSERT_PTR_NON_NULL(fringeStats, NULL); + PS_ASSERT_INT_EQUAL(fringeImages->n, fringeStats->n, NULL); + PS_ASSERT_INT_POSITIVE(nIter, NULL); + + // measure the fringe stats for the science frame and solve for the scales + pmFringeStats *scienceStats = pmFringeStatsMeasure(fringes, readout, maskVal); + + if (psTraceGetLevel("psModules.detrend") > 9) { + for (int i = 0; i < fringes->nRequested; i++) { + printf("%f", scienceStats->f->data.F32[i]); + for (int j = 0; j < fringeStats->n; j++) { + pmFringeStats *fringe = fringeStats->data[j]; + printf("\t%f", fringe->f->data.F32[i]); + } + printf("\n"); + } + } + + pmFringeScale *scale = pmFringeScaleMeasure(scienceStats, fringeStats, rej, nIter, keepFrac); + psFree(scienceStats); + + psTrace("psModules.detrend", 7, "Fringe solution:\n"); + for (int i = 0; i < fringeImages->n + 1; i++) { + psTrace("psModules.detrend", 7, "%d: %f %f\n", i, scale->coeff->data.F32[i], + scale->coeffErr->data.F32[i]); + } + + // build the fringe correction image + // XXX we could save data space by making the first image the output image + psImage *sumFringe = psImageAlloc(readout->image->numCols, readout->image->numRows, PS_TYPE_F32); + //psBinaryOp(sumFringe, sumFringe, "+", psScalarAlloc(scale->coeff->data.F32[0], PS_TYPE_F32)); + for (int i = 0; i < fringeImages->n; i++) { + + // rescale the fringe image + psBinaryOp(fringeImages->data[i], fringeImages->data[i], "*", + psScalarAlloc(scale->coeff->data.F32[i+1], PS_TYPE_F32)); + + // sum together + sumFringe = (psImage*)psBinaryOp(sumFringe, sumFringe, "+", fringeImages->data[i]); + } + psFree(scale); + + // subtract the resulting fringe frame + readout->image = (psImage*)psBinaryOp(readout->image, readout->image, "-", sumFringe); + + return sumFringe; +} + Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFringeStats.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFringeStats.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmFringeStats.h (revision 22293) @@ -0,0 +1,214 @@ +/* @file pmFringeStats.h + * @brief Measure fringe statistics, and apply correction + * + * @author Eugene Magnier, IfA + * @author Paul Price, IfA + * + * @version $Revision: 1.12 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-01-24 02:54:15 $ + * Copyright 2004-2006 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_FRINGE_STATS +#define PM_FRINGE_STATS + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// pmFringeRegions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Fringe measurement regions. +/// +/// Fringes are measured within a box of size dX,dY. A large scale smoothing is performed by subtracting the +/// background within large divisions of the image. The coordinates of the fringe points and the mask may be +/// NULL, which means that they will be generated when required. +typedef struct +{ + int nRequested; // Number of fringe points selected + int nAccepted; // Number of fringe points not masked + int dX; // Median box half-width + int dY; // Median box half-height + int nX; // Number of large-scale smoothing divisions in x (col) + int nY; // Number of large-scale smoothing divisions in y (row) + psVector *x; // Fringe point coordinates (col), or NULL + psVector *y; // Fringe point coordinates (row), or NULL + psVector *mask; // Fringe point on/off mask, or NULL +} +pmFringeRegions; + +/// Allocate fringe regions +pmFringeRegions *pmFringeRegionsAlloc (int nPts, ///< Number of fringe points to create + int dX, ///< Half-width of fringe boxes + int dY, ///< Half-height of fringe boxes + int nX, ///< Smoothing scale in x + int nY ///< Smoothing scale in y + ); + +/// Generate the fringe points +/// +/// Fringe points are generated randomly over the image. No effort is made to avoid masked regions (indeed, +/// the function knows nothing about masks). If the random number generator is NULL, then a new one will be +/// used. +bool pmFringeRegionsCreatePoints(pmFringeRegions *fringe, ///< Fringe regions to generate + const psImage *image, ///< Image for the regions (defines the size) + psRandom *random ///< Random number generator, or NULL + ); + +/// Write the regions to a FITS file +/// +/// The fringe regions are written to the FITS file, with the given extension name. The header is +/// supplemented with scalar values dX, dY, nX and nY (as PSFRNGDX, PSFRNGDY, PSFRNGNX, PSFRNGNY) from the +/// fringe regions, while the fringe coordinates and mask are written as a FITS table (as x, y, mask). +bool pmFringeRegionsWriteFits(psFits *fits, ///< Output FITS file + psMetadata *header, ///< Additional headers to write, or NULL + const pmFringeRegions *regions, ///< Regions to write + const char *extname ///< Extension name, or NULL + ); + +/// Read the regions from a FITS file +/// +/// The fringe regions are read from the FITS file, at the given extension name. The scalars are retrieved +/// from the header, while the table provides the fringe coordinates and mask. +pmFringeRegions *pmFringeRegionsReadFits(psMetadata *header, ///< Header to read, or NULL + const psFits *fits, ///< Input FITS file + const char *extname ///< Extension name, or NULL + ); + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// pmFringeStats +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Fringe measurements for a particular image +/// +/// Measurements of the median and stdev are made at each of the fringe regions. +typedef struct +{ + pmFringeRegions *regions; ///< Fringe regions + psVector *f; ///< Fringe point median + psVector *df; ///< Fringe point stdev +} +pmFringeStats; + +/// Allocate fringe statistics +pmFringeStats *pmFringeStatsAlloc(pmFringeRegions *regions // The fringe regions which will be measured + ); + +/// Measure the fringe statistics for an image +/// +/// Given an input image and fringe regions at which to measure, measures the median and stdev at each of the +/// fringe points. If the fringe points are undefined, they are generated. +pmFringeStats *pmFringeStatsMeasure(pmFringeRegions *fringe, ///< Fringe regions at which to measure + const pmReadout *readout, ///< Readout for which to measure + psMaskType maskVal ///< Mask value for image + ); + +/// Write the fringe stats for an image to a FITS table +/// +/// The fringe measurements are written to the FITS file with the given extension name. The median and stdev +/// measurements are written as a FITS table (as f and df). +bool pmFringeStatsWriteFits(psFits *fits, ///< FITS file to which to write + psMetadata *header, ///< Additional headers to write, or NULL + const pmFringeStats *fringe, ///< Fringe statistics to be written + const char *extname ///< Extension name for table + ); + +/// Read the fringe stats for an image from a FITS table +/// +/// The fringe measurements are read from the FITS file, at the given extension name. The table provides the +/// median and stdev measurements. It is assumed that the fringe measurements correspond to the regions +/// provided. +pmFringeStats *pmFringeStatsReadFits(psMetadata *header, ///< Header to read, or NULL + const psFits *fits, ///< FITS file from which to read + const char *extname, ///< Extension name to read + pmFringeRegions *regions ///< Corresponding regions + ); + +/// Concatenate the fringe stats for several readouts into a single fringe stats. +/// +/// Each readout of each chip must be measured separately (so as to avoid any gaps between the cells, as in +/// the case for GPC). But the fit must be performed with all the readouts belonging to a chip (in order to +/// get a secure measurement of the fringe amplitudes). To do so, we need to concatenate the fringe +/// measurements for each of the chip components. This function generates a new pmFringeStats from +/// concatenating those in the array. The corresponding pmFringeRegions is also generated. +pmFringeStats *pmFringeStatsConcatenate(const psArray *fringes, ///< Array of pmFringeStats for the readouts + const psVector *x0, ///< Offset in x for the readout + const psVector *y0 ///< Offset in y for the readout + ); + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Input/output for multiple pmFringeStats +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Write an array of fringes measurements to a FITS table. +/// +/// Writes an array of fringe measurements for a cell as a FITS table in the analysis metadata. The array of +/// fringe statistics must all use the same fringe regions (or there is no point in storing them all +/// together). The header is supplemented with scalar values dX, dY, nX and nY (as PSFRNGDX, PSFRNGDY, +/// PSFRNGNX, PSFRNGNY) from the fringe regions, while the fringe coordinates and mask are written as a FITS +/// table (as x, y, mask, f, df; f and df are vectors). +bool pmFringesFormat(pmCell *cell, ///< Cell for which to write + psMetadata *header, ///< Header, or NULL + const psArray *fringes ///< Array of pmFringeStats, all for the same pmFringeRegion + ); + +/// Parses an array of fringes measurements from a FITS table. +/// +/// The fringes for the cell are read from the FITS table in the analysis metadata. The table provides the +/// region and the (possibly multiple) fringe statistics for that region. The current extension is used if +/// the extension name is not provided. +psArray *pmFringesParse(pmCell *cell ///< Cell for which to read fringes + ); + + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// pmFringeScale +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// The fringe correction solution +typedef struct +{ + int nFringeFrames; ///< Number of fringe frames + psVector *coeff; ///< Fringe coefficients; size = nFringeFrames + psVector *coeffErr; ///< Error in fringe coefficients; size = nFringeFrames +} +pmFringeScale; + +/// Measure the scales for the fringe correction +/// +/// Given a fringe measurement for a science image, and an array of template fringe measurements, this +/// function measures the contribution of each of the templates to the input. Rejection is performed on the +/// fringe regions, to weed out stars etc. +pmFringeScale *pmFringeScaleMeasure(pmFringeStats *science, ///< Fringe measurements from science image + psArray *fringes, ///< Array of fringe measurements from templates + float rej, ///< Rejection threshold (in standard deviations) + unsigned int nIter, ///< Maximum number of iterations + float keepFrac ///< Minimum fraction of regions to keep + ); + +/// Allocate fringe scales +pmFringeScale *pmFringeScaleAlloc(int nFringeFrames ///< Number of fringe frames + ); + + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Fringe correction +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Solve for and apply the fringe correction +/// +/// This is a wrapper around each of the fringe correction components to measure the fringe points, solve for +/// the fringe correction, and apply the fringe correction. The input fringe images are modified (scaled by +/// the solution coefficients in order to correct the science image). Returns the summed fringe image. +psImage *pmFringeCorrect(pmReadout *in, ///< Input science image + pmFringeRegions *fringes, ///< The fringe regions used + psArray *fringeImages, ///< Fringe template images to use in correction + psArray *fringeStats, ///< Fringe stats (for templates) to use in correction + psMaskType maskVal, ///< Value to mask for science image + float rej, ///< Rejection threshold, for pmFringeScaleMeasure + unsigned int nIter, ///< Maximum number of iterations, for pmFringeScaleMeasure + float keepFrac ///< Minimum fraction of regions to keep, for pmFringeScaleMeasure + ); +/// @} +#endif Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmMaskBadPixels.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmMaskBadPixels.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmMaskBadPixels.c (revision 22293) @@ -0,0 +1,255 @@ +#if HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmHDUUtils.h" +#include "pmFPAMaskWeight.h" +#include "pmMaskBadPixels.h" + +bool pmMaskBadPixels(pmReadout *input, const pmReadout *mask, psMaskType maskVal) +{ + PS_ASSERT_PTR_NON_NULL(input, false); + PS_ASSERT_PTR_NON_NULL(input->mask, false); + PS_ASSERT_IMAGE_TYPE(input->mask, PS_TYPE_MASK, false); + + PS_ASSERT_PTR_NON_NULL(mask, false); + PS_ASSERT_PTR_NON_NULL(mask->mask, false); + PS_ASSERT_IMAGE_TYPE(mask->mask, PS_TYPE_MASK, false); + + psImage *inMask = input->mask; + psImage *exMask = mask->mask; + + // Add mask MD5 to header + pmHDU *hdu = pmHDUFromReadout(input); // HDU of interest + psVector *md5 = psImageMD5(mask->mask); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psStringPrepend(&md5string, "MASK image MD5: "); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + md5string, ""); + psFree(md5string); + + int rowMax = input->row0 + inMask->numRows; + int colMax = input->col0 + inMask->numCols; + + if (mask->row0 > input->row0 || mask->col0 > input->col0 || + mask->row0 + exMask->numRows < rowMax || mask->col0 + exMask->numCols < colMax) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Input image size exceeds that of mask image: (%d, %d) vs (%d, %d)", + inMask->numRows, inMask->numCols, exMask->numRows, exMask->numCols); + return false; + } + + // Determine total offset based on image offset with chip offset + // XXX if we choose to correct for the readout location, apply input->col0,row0 here + int offCol = input->col0 - mask->col0; + int offRow = input->row0 - mask->row0; + + // masks are both of type PS_TYPE_MASK + psMaskType **exVal = exMask->data.U8; + psMaskType **inVal = inMask->data.U8; + + // apply exMask values + if (maskVal) { + // set raised pixels in exMask which are selected by maskVal + for (int j = 0; j < inMask->numRows; j++) { + int xJ = j - offRow; + for (int i = 0; i < inMask->numCols; i++) { + int xI = i - offCol; + inVal[j][i] |= (maskVal & exVal[xJ][xI]); + } + } + } + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now, used for reporting + psString timeString = psTimeToISO(time); // String with time + psFree(time); + psStringPrepend(&timeString, "Static mask (selecting %x) applied at ", maskVal); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + timeString, ""); + psFree(timeString); + + return true; +} + + +psImage *pmMaskFlagSuspectPixels(psImage *out, const pmReadout *readout, float rej, + psMaskType maskVal) +{ + PS_ASSERT_PTR_NON_NULL(readout, NULL); + PS_ASSERT_FLOAT_LARGER_THAN(rej, 0.0, NULL); + PS_ASSERT_IMAGE_NON_NULL(readout->image, NULL); + PS_ASSERT_IMAGE_NON_EMPTY(readout->image, NULL); + PS_ASSERT_IMAGE_TYPE(readout->image, PS_TYPE_F32, NULL); + if (readout->mask) { + PS_ASSERT_IMAGE_NON_EMPTY(readout->mask, NULL); + PS_ASSERT_IMAGES_SIZE_EQUAL(readout->image, readout->mask, NULL); + PS_ASSERT_IMAGE_TYPE(readout->mask, PS_TYPE_MASK, NULL); + } + if (out) { + PS_ASSERT_IMAGE_NON_EMPTY(out, NULL); + PS_ASSERT_IMAGE_TYPE(out, PS_TYPE_S32, NULL); + PS_ASSERT_IMAGES_SIZE_EQUAL(readout->image, out, NULL); + } + + bool status; + psImage *image = readout->image; // Image of interest + psImage *mask = readout->mask; // Corresponding mask + + if (!out) { + out = psImageAlloc(image->numCols, image->numRows, PS_TYPE_S32); + psImageInit(out, 0); + } + + bool whole = false; // Mask whole image? + float median = psMetadataLookupF32 (&status, readout->analysis, "READOUT.MEDIAN"); + if (!status || !isfinite(median)) { + whole = true; + } + float stdev = psMetadataLookupF32 (&status, readout->analysis, "READOUT.STDEV"); + if (!status || !isfinite(stdev)) { + whole = true; + } + + + psTrace ("psModules.detrend", 3, "suspect: %f +/- %f\n", median, stdev); + + if (whole) { + // If we get down here and the statistics are missing, then we should go and mask the entire image + psWarning("Missing statistics --- flagging entire image as suspect."); + return (psImage*)psBinaryOp(out, out, "+", psScalarAlloc(1.0, PS_TYPE_S32)); + } + + for (int y = 0; y < image->numRows; y++) { + for (int x = 0; x < image->numCols; x++) { + if (fabs((image->data.F32[y][x] - median) / stdev) >= rej && + (!mask || !(mask->data.PS_TYPE_MASK_DATA[y][x] & maskVal))) { + out->data.S32[y][x]++; + } + } + } + + return out; +} + +psImage *pmMaskIdentifyBadPixels(const psImage *suspects, psMaskType maskVal, int nTotal, float thresh, pmMaskIdentifyMode mode) +{ + PS_ASSERT_IMAGE_NON_NULL(suspects, NULL); + PS_ASSERT_IMAGE_NON_EMPTY(suspects, NULL); + PS_ASSERT_IMAGE_TYPE(suspects, PS_TYPE_S32, NULL); + + float limit = NAN; // Limit for masking + + switch (mode) { + case PM_MASK_ID_VALUE: + limit = thresh; + break; + + case PM_MASK_ID_FRACTION: + limit = thresh * nTotal; + break; + + case PM_MASK_ID_SIGMA: { + psStats *stats = psStatsAlloc(PS_STAT_CLIPPED_STDEV); // Statistics + stats->clipSigma = 5.0; + stats->clipIter = 1; + if (!psImageStats(stats, suspects, NULL, 0) || !isfinite(stats->clippedStdev)) { + psError(PS_ERR_UNKNOWN, false, "Unable to perform statistics.\n"); + psFree(stats); + return NULL; + } + limit = thresh * stats->clippedStdev; + psTrace ("psModules.detrend", 3, "bad: %f -> %f\n", stats->clippedStdev, limit); + psFree(stats); + break; + } + + case PM_MASK_ID_POISSON: { + psStats *stats = psStatsAlloc(PS_STAT_MAX); // Statistics + if (!psImageStats(stats, suspects, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "Unable to perform statistics.\n"); + psFree(stats); + return NULL; + } + psHistogram *histo = psHistogramAlloc(-0.5, stats->max + 0.5, stats->max + 1); + psFree(stats); + if (!psImageHistogram(histo, suspects, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate histogram.\n"); + psFree(histo); + return NULL; + } + + // Find the mode. Since this is a Poisson distribution (more or less), this should also be the mean + // and variance. + int max = 0; // Index of the mode + for (int i = 0; i < histo->nums->n; i++) { + if (histo->nums->data.F32[i] > histo->nums->data.F32[max]) { + max = i; + } + } + + // Since the mode is most likely zero, we add one to get something realistic. Then "thresh" is + // negative, so we subtract instead of add. + limit = max + 1.0 - thresh * sqrtf((float)max + 1.0); + + psTrace ("psModules.detrend", 3, "bad: mode: %d, stdev: %f, limit: %f\n", max, sqrtf((float)max + 1.0), limit); + break; + } + default: + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Invalid mask identify mode"); + return NULL; + } + + psImage *badpix = psImageAlloc(suspects->numCols, suspects->numRows, PS_TYPE_MASK); // Bad pixel mask + psImageInit(badpix, 0); + + if (psTraceGetLevel("psModules.detrend") > 9) { + psStats *stats = psStatsAlloc(PS_STAT_MIN | PS_STAT_MAX); // Statistics + psImageStats(stats, suspects, NULL, 0); + psHistogram *histo = psHistogramAlloc(-0.5, stats->max + 0.5, stats->max + 1); + psImageHistogram(histo, suspects, NULL, 0); + for (int i = 0; i < histo->nums->n; i++) { + printf("%f --> %f : %f\n", histo->bounds->data.F32[i], histo->bounds->data.F32[i + 1], + histo->nums->data.F32[i]); + } + psFree(stats); + psFree(histo); + printf("Threshold: %f\n", limit); + } + + psTrace ("psModules.detrend", 3, "bad pixel threshold: %f", limit); + + for (int y = 0; y < suspects->numRows; y++) { + for (int x = 0; x < suspects->numCols; x++) { + if (suspects->data.S32[y][x] >= limit) { + badpix->data.PS_TYPE_MASK_DATA[y][x] = maskVal; + } + } + } + + return badpix; +} + +pmMaskIdentifyMode pmMaskIdentifyModeFromString (const char *string) { + + if (!strcasecmp (string, "VALUE")) { + return PM_MASK_ID_VALUE; + } + if (!strcasecmp (string, "FRACTION")) { + return PM_MASK_ID_FRACTION; + } + if (!strcasecmp (string, "SIGMA")) { + return PM_MASK_ID_SIGMA; + } + if (!strcasecmp (string, "POISSON")) { + return PM_MASK_ID_POISSON; + } + return PM_MASK_ID_NONE; +} Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmMaskBadPixels.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmMaskBadPixels.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmMaskBadPixels.h (revision 22293) @@ -0,0 +1,67 @@ +/* @file pmMaskBadPixels.h + * @brief Mask bad pixels + * + * @author Ross Harman, MHPCC + * @author Eugene Magnier, IfA + * + * @version $Revision: 1.15 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-12-24 21:10:28 $ + * Copyright 2004 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_MASK_BAD_PIXELS_H +#define PM_MASK_BAD_PIXELS_H + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +typedef enum { + PM_MASK_ID_NONE, + PM_MASK_ID_VALUE, + PM_MASK_ID_FRACTION, + PM_MASK_ID_SIGMA, + PM_MASK_ID_POISSON, +} pmMaskIdentifyMode; + +pmMaskIdentifyMode pmMaskIdentifyModeFromString (const char *string); + +/// Applies the bad pixel mask to the input +/// +/// Pixels marked as bad within the mask are marked as bad within the input image's mask. If maskVal is +/// non-zero, all pixels in the mask have any of the same bits sets as maskVal shall have the corresponding +/// bits raised. If maskVal is zero, any zero pixels in the mask are OR-ed with PM_MASK_BAD. Position +/// offsets (such as due to trimming) between the input and mask are applied so that the same pixels are +/// referred to. The science readout must already have a supplied mask element (use eg. pmReadoutSetMask). +/// The supplied mask image must be of MASK type +bool pmMaskBadPixels(pmReadout *input, ///< Input science image + const pmReadout *mask, ///< Mask image to apply + psMaskType maskVal ///< Mask value to apply + ); + +/// Find pixels outlying from the background, flagging suspect pixels +/// +/// Pixels more than "rej" standard deviations from the background level (in flat-fielded, +/// background-subtracted images) have the corresponding pixel in the "suspect pixels" image +/// incremented. After accumulating over a suitable sample of images, bad pixels should have a +/// high value in the suspect pixels image, allowing them to be identified. The suspect pixels +/// image is of type S32. The relevant median and standard deviation must be supplied in the +/// readout->analysis metadata as READOUT.MEDIAN, READOUT.STDEVe +psImage *pmMaskFlagSuspectPixels(psImage *out, ///< Suspected bad pixels image, or NULL + const pmReadout *readout, ///< Readout to inspect + float rej, ///< Rejection threshold (standard deviations) + psMaskType maskVal ///< Mask value for statistics + ); + +/// Identify bad pixels from the suspect pixels image +/// +/// Bad pixels are identified from the suspect pixels image (accumulated over a large number of images). +/// Pixels marked as suspect in more than "thresh" standard deviations from the mean are identified as bad +/// pixels (output image). If "thresh" is negative, a Poisson is assumed. +psImage *pmMaskIdentifyBadPixels(const psImage *suspects, ///< Accumulated suspect pixels image + psMaskType maskVal, ///< Value to set for bad pixels + int nTotal, + float thresh, ///< Threshold for bad pixel (standard deviations) + pmMaskIdentifyMode mode + ); +/// @} +#endif Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmNonLinear.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmNonLinear.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmNonLinear.c (revision 22293) @@ -0,0 +1,93 @@ +#if HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmNonLinear.h" + +pmReadout *pmNonLinearityPolynomial(pmReadout *inputReadout, const psPolynomial1D *input1DPoly) +{ + PS_ASSERT_PTR_NON_NULL(inputReadout, NULL); + PS_ASSERT_PTR_NON_NULL(inputReadout->image, NULL); + PS_ASSERT_IMAGE_TYPE(inputReadout->image, PS_TYPE_F32, NULL); + PS_ASSERT_PTR_NON_NULL(input1DPoly, NULL); + + psImage *image = inputReadout->image; // Image to correct + for (int i = 0; i < image->numRows; i++) { + for (int j = 0; j < image->numCols; j++) { + image->data.F32[i][j] = psPolynomial1DEval(input1DPoly, image->data.F32[i][j]); + } + } + return inputReadout; +} + +// set the bin closest to the corresponding value. +#define PS_BIN_FOR_VALUE(RESULT, VECTOR, VALUE) { \ + psVectorBinaryDisectResult result; \ + psScalar tmpScalar; \ + tmpScalar.type.type = PS_TYPE_F32; \ + tmpScalar.data.F32 = (VALUE); \ + RESULT = psVectorBinaryDisect (&result, VECTOR, &tmpScalar); \ + switch (result) { \ + case PS_BINARY_DISECT_PASS: \ + break; \ + case PS_BINARY_DISECT_OUTSIDE_RANGE: \ + numPixels ++; \ + break; \ + case PS_BINARY_DISECT_INVALID_INPUT: \ + case PS_BINARY_DISECT_INVALID_TYPE: \ + psAbort ("programming error"); \ + break; \ + } } + +pmReadout *pmNonLinearityLookup(pmReadout *inputReadout, const psVector *inFlux, const psVector *outFlux) +{ + PS_ASSERT_PTR_NON_NULL(inputReadout, NULL); + PS_ASSERT_PTR_NON_NULL(inputReadout->image, NULL); + PS_ASSERT_IMAGE_TYPE(inputReadout->image, PS_TYPE_F32, NULL); + PS_ASSERT_PTR_NON_NULL(inFlux, NULL); + if (inFlux->n < 2) { + psError(PS_ERR_UNKNOWN, true, + "pmNonLinearityLookup(): input vector less than 2 elements. Returning inputReadout image."); + return(inputReadout); + } + PS_ASSERT_PTR_NON_NULL(outFlux,NULL); + psS32 tableSize = inFlux->n; + if (inFlux->n != outFlux->n) { + tableSize = PS_MIN(inFlux->n, outFlux->n); + psLogMsg(__func__, PS_LOG_WARN, + "WARNING: pmNonLinear.c: pmNonLinearityLookup(): " + "input vectors have different sizes (%ld, %ld)\n", + inFlux->n, outFlux->n); + } + PS_ASSERT_VECTOR_TYPE(inFlux, PS_TYPE_F32, NULL); + PS_ASSERT_VECTOR_TYPE(outFlux, PS_TYPE_F32, NULL); + + psImage *image = inputReadout->image; // Input image + int numPixels = 0; // Number of pixels outside the range + int binNum; + + for (int i = 0; i < image->numRows; i++) { + for (int j = 0; j < image->numCols; j++) { + float value = image->data.F32[i][j]; + PS_BIN_FOR_VALUE(binNum, inFlux, value); + + // Perform linear interpolation. + // XXX this will result in non-sensical results if inFlux contains equal-value + // bins. either enforce d(inFlux)/d(binNum) > 0 or see psStats.c PS_BIN_INTERPOLATE + float slope = + (outFlux->data.F32[binNum + 1] - outFlux->data.F32[binNum]) / + (inFlux->data.F32[binNum + 1] - inFlux->data.F32[binNum]); + image->data.F32[i][j] = slope*(value - inFlux->data.F32[binNum]) + outFlux->data.F32[binNum]; + } + } + if (numPixels > 0) { + psLogMsg(__func__, PS_LOG_WARN, + "WARNING: pmNonLinear.c: pmNonLinearityLookup(): %d pixels outside table.", numPixels); + } + return inputReadout; +} Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmNonLinear.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmNonLinear.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmNonLinear.h (revision 22293) @@ -0,0 +1,35 @@ +/* @file pmNonLinear.h + * @brief Perform non-linear correction through polynomial or table lookup + * + * @author George Gusciora, MHPCC + * @author Paul Price, IfA + * + * @version $Revision: 1.6 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-03-30 21:12:56 $ + * Copyright 2004 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_NON_LINEAR_H +#define PM_NON_LINEAR_H + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +/// Correct non-linearity through polynomial +/// +/// Applies a polynomial to the flux of each pixel in the input image to determine the corrected flux. +pmReadout *pmNonLinearityPolynomial(pmReadout *in, ///< Input image, to correct + const psPolynomial1D *coeff ///< Polynomial for non-linearity correction + ); + +/// Correct non-linearity through table lookup +/// +/// For each pixel in the input image, performs linear interpolation on the table (from the two vectors) to +/// determine the corrected flux. +pmReadout *pmNonLinearityLookup(pmReadout *in, ///< Input image, to correct + const psVector *inFlux, ///< Table column with input fluxes + const psVector *outFlux ///< Table column with output fluxes + ); + +/// @} +#endif Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmOverscan.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmOverscan.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmOverscan.c (revision 22293) @@ -0,0 +1,401 @@ +#if HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmHDUUtils.h" +#include "pmFPALevel.h" +#include "pmFPAview.h" +#include "pmFPACalibration.h" + +#include "pmOverscan.h" + +#define SMOOTH_NSIGMA 4.0 // Number of Gaussian sigma the smoothing kernel extends + +static void pmOverscanOptionsFree(pmOverscanOptions *options) +{ + psFree(options->stat); + psFree(options->poly); + psFree(options->spline); +} + +pmOverscanOptions *pmOverscanOptionsAlloc(bool single, pmFit fitType, unsigned int order, psStats *stat, + int boxcar, float gauss) +{ + pmOverscanOptions *opts = psAlloc(sizeof(pmOverscanOptions)); + psMemSetDeallocator(opts, (psFreeFunc)pmOverscanOptionsFree); + + // Inputs + opts->single = single; + opts->constant = false; + opts->fitType = fitType; + opts->order = order; + opts->stat = psMemIncrRefCounter(stat); + + // Smoothing + opts->boxcar = boxcar; + opts->gauss = gauss; + + // Outputs + opts->poly = NULL; + opts->spline = NULL; + + return opts; +} + +// Produce an overscan vector from an array of pixels +psVector *pmOverscanVector(float *chi2, // chi^2 from fit + pmOverscanOptions *overscanOpts, // Overscan options + const psArray *pixels, // Array of vectors containing the pixel values + psStats *myStats // Statistic to use in reducing the overscan + ) +{ + assert(overscanOpts); + assert(pixels); + assert(myStats); + + psStatsOptions statistic = psStatsSingleOption(myStats->options); // Statistic to use + assert(statistic != 0); + + // Reduce the overscans + psVector *reduced = psVectorAlloc(pixels->n, PS_TYPE_F32); // Overscan for each row + psVector *ordinate = psVectorAlloc(pixels->n, PS_TYPE_F32); // Ordinate + psVector *mask = psVectorAlloc(pixels->n, PS_TYPE_U8); // Mask for fitting + + for (int i = 0; i < pixels->n; i++) { + psVector *values = pixels->data[i]; // Vector with overscan values + if (values->n > 0) { + mask->data.U8[i] = 0; + ordinate->data.F32[i] = 2.0*(float)i/(float)pixels->n - 1.0; // Scale to [-1,1] + psVectorStats(myStats, values, NULL, NULL, 0); + reduced->data.F32[i] = psStatsGetValue(myStats, statistic); + } else if (overscanOpts->fitType == PM_FIT_NONE) { + psError(PS_ERR_UNKNOWN, true, "The overscan is not supplied for all points on the " + "image, and no fit is requested.\n"); + return NULL; + } else { + // We'll fit this one out + mask->data.U8[i] = 1; + } + } + + // Smooth the reduced vector + if (overscanOpts->boxcar > 0) { + psVector *smoothed = psVectorBoxcar(NULL, reduced, overscanOpts->boxcar); // Smoothed vector + psFree(reduced); + reduced = smoothed; + } + if (isfinite(overscanOpts->gauss) && overscanOpts->gauss > 0) { + if (overscanOpts->boxcar > 0) { + psWarning("Gaussian smoothing the boxcar smoothed overscan --- you asked for it."); + } + psVector *smoothed = psVectorSmooth(NULL, reduced, overscanOpts->gauss, SMOOTH_NSIGMA); + psFree(reduced); + reduced = smoothed; + } + + // Fit the overscan, if required + psVector *fitted; // Fitted overscan values + switch (overscanOpts->fitType) { + case PM_FIT_NONE: + // No fitting --- that's easy. + fitted = psMemIncrRefCounter(reduced); + break; + case PM_FIT_POLY_ORD: + if (overscanOpts->poly && (overscanOpts->poly->nX != overscanOpts->order || + overscanOpts->poly->type != PS_POLYNOMIAL_ORD)) { + psFree(overscanOpts->poly); + overscanOpts->poly = NULL; + } + if (! overscanOpts->poly) { + overscanOpts->poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, overscanOpts->order); + } + psVectorFitPolynomial1D(overscanOpts->poly, mask, 1, reduced, NULL, ordinate); + fitted = psPolynomial1DEvalVector(overscanOpts->poly, ordinate); + break; + case PM_FIT_POLY_CHEBY: + if (overscanOpts->poly && (overscanOpts->poly->nX != overscanOpts->order || + overscanOpts->poly->type != PS_POLYNOMIAL_CHEB)) { + psFree(overscanOpts->poly); + overscanOpts->poly = NULL; + } + if (! overscanOpts->poly) { + overscanOpts->poly = psPolynomial1DAlloc(PS_POLYNOMIAL_CHEB, overscanOpts->order); + } + psVectorFitPolynomial1D(overscanOpts->poly, mask, 1, reduced, NULL, ordinate); + fitted = psPolynomial1DEvalVector(overscanOpts->poly, ordinate); + break; + case PM_FIT_SPLINE: + // XXX I don't think psSpline1D is up to scratch yet --- it has no mask, and requires an + // input spline + overscanOpts->spline = psVectorFitSpline1D(reduced, ordinate); + fitted = psSpline1DEvalVector(overscanOpts->spline, ordinate); + break; + default: + psError(PS_ERR_UNKNOWN, true, "Unknown value for the fitting type: %d\n", overscanOpts->fitType); + return NULL; + break; + } + + if (chi2) { + *chi2 = 0.0; // chi^2 (sort of) + for (int i = 0; i < reduced->n; i++) { + *chi2 += PS_SQR(fitted->data.F32[i] - reduced->data.F32[i]); + } + } + + psFree(reduced); + psFree(ordinate); + psFree(mask); + + return fitted; +} + +bool pmOverscanUpdateHeader (pmHDU *hdu, pmOverscanOptions *overscanOpts, float chi2) { + + psString comment = NULL; // Comment to add + + switch (overscanOpts->fitType) { + case PM_FIT_POLY_ORD: + case PM_FIT_POLY_CHEBY: { + psStringAppend(&comment, "Overscan fit (chi2: %.2f): ", chi2); + psPolynomial1D *poly = overscanOpts->poly; // The polynomial + for (int i = 0; i < poly->nX; i++) { + psStringAppend(&comment, "%.1f ", poly->coeff[i]); + } + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + comment = NULL; + + // write metadata header value + psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, + "Overscan value", poly->coeff[0]); + psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, + "Overscan stdev", poly->coeffErr[0]); + break; + } + case PM_FIT_SPLINE: { + psSpline1D *spline = overscanOpts->spline; // The spline + for (int i = 0; i < spline->n; i++) { + psStringAppend(&comment, "Overscan fit (chi2: %.2f) %d:", chi2, i); + psPolynomial1D *poly = spline->spline[i]; // i-th polynomial + for (int j = 0; j < poly->nX; j++) { + psStringAppend(&comment, "%.1f ", poly->coeff[i]); + } + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + comment, ""); + psFree(comment); + comment = NULL; + } + // write metadata header value + psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, + "Overscan value", NAN); + psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, + "Overscan stdev", NAN); + break; + } + case PM_FIT_NONE: + break; + default: + psAbort("Should never get here!!!\n"); + } + return true; +} + +bool pmOverscanSubtract (pmReadout *input, pmOverscanOptions *overscanOpts) { + + assert (input); + + if (overscanOpts == NULL) return true; // no overscan subtraction requested + + pmHDU *hdu = pmHDUFromReadout(input); // HDU of interest + psImage *image = input->image; + + // check for 'soft bias' (simple, fixed offset to be subtracted) + if (overscanOpts->constant) { + // write metadata header value + psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan value", + overscanOpts->value); + psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", NAN); + + (void)psBinaryOp(input->image, input->image, "-", psScalarAlloc((float)overscanOpts->value, PS_TYPE_F32)); + + return true; + } + + // we are performing a statitical analysis of the overscan region + + // Check for an unallowable pmFit. + if (overscanOpts->fitType != PM_FIT_NONE && overscanOpts->fitType != PM_FIT_POLY_ORD && + overscanOpts->fitType != PM_FIT_POLY_CHEBY && overscanOpts->fitType != PM_FIT_SPLINE) { + psError(PS_ERR_UNKNOWN, true, "Invalid fit type (%d). Returning original image.\n", + overscanOpts->fitType); + return false; + } + + psList *overscans = input->bias; // List of the overscan images + + psStatsOptions statistic = psStatsSingleOption(overscanOpts->stat->options); // Statistic to use + if (statistic == 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Multiple or no statistics options set: %p\n", + overscanOpts->stat); + return false; + } + psStats *stats = psStatsAlloc(statistic); // A new psStats, to avoid clobbering original + + psString comment = NULL; // Comment to add + psStringAppend(&comment, "Subtracting overscan (stat %x; type %x; order %d)", + statistic, overscanOpts->fitType, overscanOpts->order); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + comment, ""); + psFree(comment); + + // Reduce all overscan pixels to a single value + if (overscanOpts->single) { + psVector *pixels = psVectorAlloc(0, PS_TYPE_F32); + psListIterator *iter = psListIteratorAlloc(overscans, PS_LIST_HEAD, false); // Iterator + psImage *overscan = NULL; // Overscan image from iterator + while ((overscan = psListGetAndIncrement(iter))) { + int index = pixels->n; // Index + pixels = psVectorRealloc(pixels, pixels->n + overscan->numRows * overscan->numCols); + pixels->n += overscan->numRows * overscan->numCols; + for (int i = 0; i < overscan->numRows; i++) { + memcpy(&pixels->data.F32[index], overscan->data.F32[i], + overscan->numCols * sizeof(psF32)); + index += overscan->numCols; + } + } + psFree(iter); + + (void)psVectorStats(stats, pixels, NULL, NULL, 0); + psFree(pixels); + double reduced = psStatsGetValue(stats, statistic); // Result of statistics + + psString comment = NULL; // Comment to add + psStringAppend(&comment, "Overscan value: %f", reduced); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + + // write metadata header value + psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan value", + reduced); + psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", NAN); + + (void)psBinaryOp(image, image, "-", psScalarAlloc((float)reduced, PS_TYPE_F32)); + psFree(stats); + return true; + } + + // We are performing a row-by-row overscan subtraction + bool readRows = psMetadataLookupBool(NULL, input->parent->concepts, "CELL.READDIR"); // Read direction + float chi2 = NAN; // chi^2 from fit + + // adjust operation depending on the read direction : need to re-org pixels for columns + if (readRows) { + // The read direction is rows + psArray *pixels = psArrayAlloc(image->numRows); // Array of vectors containing pixels + for (int i = 0; i < pixels->n; i++) { + pixels->data[i] = psVectorAlloc(0, PS_TYPE_F32); + } + + // Pull the pixels out into the vectors + psListIterator *iter = psListIteratorAlloc(overscans, PS_LIST_HEAD, false); // Iterator + psImage *overscan = NULL; // Overscan image from iterator + while ((overscan = psListGetAndIncrement(iter))) { + // the overscan and image might not be aligned. pixels->data represents + // the image row pixels. + int diff = overscan->row0 - image->row0; // Offset between the two regions + for (int i = PS_MAX(0,diff); i < PS_MIN(image->numRows, overscan->numRows + diff); i++) { + int j = i - diff; + // i is row on image + // j is row on overscan + psVector *values = pixels->data[i]; + int index = values->n; // Index in the vector + values = psVectorRealloc(values, values->n + overscan->numCols); + values->n += overscan->numCols; + memcpy(&values->data.F32[index], overscan->data.F32[j], + overscan->numCols * PSELEMTYPE_SIZEOF(PS_TYPE_F32)); + index += overscan->numCols; + pixels->data[i] = values; // Update the pointer in case it's moved + } + } + psFree(iter); + + // Reduce the overscans + psVector *reduced = pmOverscanVector(&chi2, overscanOpts, pixels, stats); + psFree(pixels); + if (! reduced) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate overscan vector.\n"); + psFree(stats); + return false; + } + + // Subtract row by row + for (int i = 0; i < image->numRows; i++) { + for (int j = 0; j < image->numCols; j++) { + image->data.F32[i][j] -= reduced->data.F32[i]; + } + } + psFree(reduced); + } else { + // The read direction is columns + psArray *pixels = psArrayAlloc(image->numCols); // Array of vectors containing pixels + for (int i = 0; i < pixels->n; i++) { + psVector *values = psVectorAlloc(0, PS_TYPE_F32); + pixels->data[i] = values; + } + + // Pull the pixels out into the vectors + psListIterator *iter = psListIteratorAlloc(overscans, PS_LIST_HEAD, false); // Iterator + psImage *overscan = NULL; // Overscan image from iterator + while ((overscan = psListGetAndIncrement(iter))) { + // the overscan and image might not be aligned. pixels->data represents + // the image row pixels. + int diff = overscan->col0 - image->col0; // Offset between the two regions + for (int i = PS_MAX(0,diff); i < PS_MIN(image->numCols, overscan->numCols + diff); i++) { + int iFixed = i - diff; + // i is column on image + // iFixed is column on overscan + psVector *values = pixels->data[i]; + int index = values->n; // Index in the vector + values = psVectorRealloc(values, values->n + overscan->numRows); + for (int j = 0; j < overscan->numRows; j++) { + values->data.F32[index++] = overscan->data.F32[j][iFixed]; + } + values->n += overscan->numRows; + pixels->data[i] = values; // Update the pointer in case it's moved + } + } + psFree(iter); + + // Reduce the overscans + psVector *reduced = pmOverscanVector(&chi2, overscanOpts, pixels, stats); + psFree(pixels); + if (! reduced) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate overscan vector.\n"); + psFree(stats); + return false; + } + + // Subtract column by column + for (int i = 0; i < image->numCols; i++) { + for (int j = 0; j < image->numRows; j++) { + image->data.F32[j][i] -= reduced->data.F32[i]; + } + } + psFree(reduced); + } + + pmOverscanUpdateHeader (hdu, overscanOpts, chi2); + psFree(stats); + + return true; + +} // End of overscan subtraction + Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmOverscan.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmOverscan.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmOverscan.h (revision 22293) @@ -0,0 +1,72 @@ +/* @file pmOverscan.h + * @brief Functions to subtract the overscan, used by pmBiasSubtract + * + * @author George Gusciora, MHPCC + * @author Paul Price, IfA + * @author Eugene Magnier, IfA + * + * @version $Revision: 1.2 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-08-15 20:21:18 $ + * Copyright 2004--2006 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_OVERSCAN_H +#define PM_OVERSCAN_H + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +/// Type of fit to perform +typedef enum { + PM_FIT_NONE, ///< No fit + PM_FIT_POLY_ORD, ///< Fit ordinary polynomial + PM_FIT_POLY_CHEBY, ///< Fit Chebyshev polynomial + PM_FIT_SPLINE ///< Fit cubic splines +} pmFit; + +/// Options for overscan subtraction +/// +/// The overscan subtraction may be performed by reducing all overscan regions to a single value (e.g., if +/// there is no structure); or the overscan may be fit perpendicular to the read direction (usually the +/// columns) with a particular functional form; or a single value may be subtracted for each read/scan without +/// fitting (if the structure defies characterisation). In any case, statistics are required to reduce +/// multiple values to a single value (either for the scan, or for the entire overscan regions). +typedef struct +{ + // Inputs + bool single; ///< Reduce all overscan regions to a single value? + bool constant; ///< use a supplied constant value (do not measure region) + float value; ///< supplied value if needed (per above) + pmFit fitType; ///< Type of fit to overscan + unsigned int order; ///< Order of polynomial, or number of spline pieces + psStats *stat; ///< Statistic to use when reducing the minor direction + int boxcar; ///< Boxcar smoothing radius + float gauss; ///< Gaussian smoothing sigma + // Outputs + psPolynomial1D *poly; ///< Result of polynomial fit + psSpline1D *spline; ///< Result of spline fit +} +pmOverscanOptions; + +/// Allocator for overscan options +pmOverscanOptions *pmOverscanOptionsAlloc(bool single, ///< Reduce all overscan regions to a single value? + pmFit fitType, ///< Type of fit to overscan + unsigned int order, ///< Order of polynomial, or number of splines + psStats *stat, ///< Statistic to use + int boxcar, ///< Boxcar smoothing radius + float gauss ///< Gaussian smoothing sigma + ); + +psVector *pmOverscanVector(float *chi2, // chi^2 from fit + pmOverscanOptions *overscanOpts, // Overscan options + const psArray *pixels, // Array of vectors containing the pixel values + psStats *myStats // Statistic to use in reducing the overscan + ); + +bool pmOverscanUpdateHeader (pmHDU *hdu, pmOverscanOptions *overscanOpts, float chi2); + +bool pmOverscanSubtract (pmReadout *input, pmOverscanOptions *overscanOpts); + +/// @} +#endif + Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmShifts.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmShifts.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmShifts.c (revision 22293) @@ -0,0 +1,477 @@ +#if HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmFPALevel.h" +#include "pmFPAUtils.h" +#include "pmShifts.h" + +#define SHIFTS_BUFFER 100 // Buffer size for shifts +#define TRACE "psModules.detrend" // Trace facility +#define FFT_SIZE 25 // Size at which we use FFT instead of direct convolution + +// XXX To do: +// * Make the table column names configurable, by having a SHIFTS metadata in the camera format, with entries +// specifying the column names. + + +static void shiftsFree(pmShifts *shifts) +{ + psFree(shifts->x); + psFree(shifts->y); + psFree(shifts->t); + return; +} + +pmShifts *pmShiftsAlloc(bool tRel, bool xyRel) +{ + pmShifts *shifts = psAlloc(sizeof(pmShifts)); + psMemSetDeallocator(shifts, (psFreeFunc)shiftsFree); + + shifts->x = psVectorAllocEmpty(SHIFTS_BUFFER, PS_TYPE_S32); + shifts->y = psVectorAllocEmpty(SHIFTS_BUFFER, PS_TYPE_S32); + shifts->t = psVectorAllocEmpty(SHIFTS_BUFFER, PS_TYPE_F32); + shifts->num = 0; + + // Suitable defaults + shifts->tRelative = tRel; + shifts->xyRelative = xyRel; + + return shifts; +} + +// Look up the cell within a hash; supplement the hash with a new value if it doesn't exist +static pmShifts *cellVectors(psHash *shifts, // Hash of shifts + const char *cellName, // Key for hash + bool tRel, bool xyRel // Are the shifts relative? + ) +{ + assert(shifts); + assert(cellName); + + // Find the appropriate cell + pmShifts *vectors = psHashLookup(shifts, cellName); + if (!vectors) { + vectors = pmShiftsAlloc(tRel, xyRel); + psHashAdd(shifts, cellName, vectors); + psFree(vectors); // Drop reference + } + return vectors; +} + + +bool pmShiftsRead(const pmCell *cell, psFits *fits) +{ + PS_ASSERT_PTR_NON_NULL(fits, false); + PS_ASSERT_PTR_NON_NULL(cell, false); + + bool mdok; // Status of MD lookup + pmShifts *check = psMetadataLookupPtr(&mdok, cell->analysis, PM_SHIFTS_TABLE_NAME); // Table, or NULL + if (check) { + psTrace(TRACE, 2, "Cell already has OT kernel present.\n"); + return true; // No error + } + psTrace(TRACE, 2, "Reading FITS file for OT kernels.\n"); + + // Determine camera layout + pmChip *chip = cell->parent; // The parent chip + pmFPA *fpa = chip->parent; // The parent FPA + pmHDU *phu = NULL; // The primary header + pmFPALevel phuLevel = PM_FPA_LEVEL_NONE; // Level of the PHU + long numChips = 0; // Number of chips below the PHU; for setting efficient hash size + long numCells = 0; // Number of cells below the PHU; for setting efficient hash size + if (fpa->hdu) { + phu = fpa->hdu; + // Count the cells + psArray *chips = fpa->chips; // Array of chips + numChips = chips->n; + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + numCells += chip->cells->n; + } + numCells = (float)numCells / (float)numChips + 0.5; // Average number of cells per chip + phuLevel = PM_FPA_LEVEL_FPA; + } + if (!phu && chip->hdu) { + phu = chip->hdu; + numChips = 1; + numCells = chip->cells->n; + phuLevel = PM_FPA_LEVEL_CHIP; + } + if (!phu && cell->hdu) { + phu = cell->hdu; + numChips = 0; + numCells = 1; + phuLevel = PM_FPA_LEVEL_CELL; + } + if (!phu || phuLevel == PM_FPA_LEVEL_NONE) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Can't find the PHU.\n"); + return false; + } + + + // Find out what to read + psMetadata *shiftsInfo = psMetadataLookupMetadata(&mdok, phu->format, "SHIFTS"); // Shifts information + if (!mdok || !shiftsInfo) { + // We have read all the shifts that we have been told about --- which is none. + return true; + } + const char *shiftsExt = psMetadataLookupStr(&mdok, shiftsInfo, "EXTENSION"); // Extension name for shifts + if (!mdok || !shiftsExt) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Can't find EXTENSION name in SHIFTS information from camera format."); + return false; + } + const char *chipCol = NULL; // Column name for chip + if (phuLevel == PM_FPA_LEVEL_FPA) { + chipCol = psMetadataLookupStr(&mdok, shiftsInfo, "CHIP"); + if (!mdok || !chipCol) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Can't find CHIP column name in SHIFTS information from camera format."); + return false; + } + } + const char *cellCol = NULL; // Column name for cell + if (phuLevel <= PM_FPA_LEVEL_CHIP) { + cellCol = psMetadataLookupStr(&mdok, shiftsInfo, "CELL"); + if (!mdok || !chipCol) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Can't find CELL column name in SHIFTS information from camera format."); + return false; + } + } + const char *tCol = psMetadataLookupStr(&mdok, shiftsInfo, "T"); + if (!mdok || !tCol) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Can't find T column name in SHIFTS information from camera format."); + return false; + } + const char *xCol = psMetadataLookupStr(&mdok, shiftsInfo, "X"); + if (!mdok || !xCol) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Can't find X column name in SHIFTS information from camera format."); + return false; + } + const char *yCol = psMetadataLookupStr(&mdok, shiftsInfo, "Y"); + if (!mdok || !yCol) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Can't find Y column name in SHIFTS information from camera format."); + return false; + } + + bool tRel = psMetadataLookupBool(&mdok, shiftsInfo, "TRELATIVE"); + if (!mdok) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Can't find TRELATIVE in SHIFTS information from camera format."); + return false; + }; + bool xyRel = psMetadataLookupStr(&mdok, shiftsInfo, "XYRELATIVE"); + if (!mdok) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Can't find XYRELATIVE in SHIFTS information from camera format."); + return false; + }; + + // Read the FITS file + int origExt = psFitsGetExtNum(fits); // Original extension number; to preserve position + if (!psFitsMoveExtName(fits, shiftsExt)) { + psError(PS_ERR_IO, false, "Unable to move to shifts extension %s", shiftsExt); + return false; + } + psArray *table = psFitsReadTable(fits); // The table of shifts + if (!table) { + psError(PS_ERR_IO, false, "Unable to read shifts table.\n"); + return false; + } + + // More sensible storage + pmShifts *singleShifts = NULL; // Shifts for a single cell + psHash *multipleShifts = NULL; // Shifts for multiple cells, stored by cell name + switch (phuLevel) { + case PM_FPA_LEVEL_FPA: + multipleShifts = psHashAlloc(2 * numChips); + break; + case PM_FPA_LEVEL_CHIP: + multipleShifts = psHashAlloc(2 * numCells); + break; + case PM_FPA_LEVEL_CELL: + singleShifts = pmShiftsAlloc(tRel, xyRel); + break; + default: + psAbort("Should never get here.\n"); + } + + // Pull values out of the table into something a bit more sensible + for (int i = 0; i < table->n; i++) { + psMetadata *row = table->data[i]; // The row of interest + + const char *chipName = NULL; // Name of chip + if (phuLevel == PM_FPA_LEVEL_FPA) { + // Only care about the chip name if there's more than one chip + chipName = psMetadataLookupStr(&mdok, row, chipCol); + if (!mdok || !chipName) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find column %s in row %d of shifts table\n", + chipCol, i); + psFree(multipleShifts); + psFree(table); + return false; + } + } + const char *cellName = NULL; // Name of cell + if (phuLevel <= PM_FPA_LEVEL_CHIP) { + // Only care about the cell name if there's a chip + cellName = psMetadataLookupStr(&mdok, row, cellCol); + if (!mdok || !cellName) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find column %s in row %d of shifts table\n", + cellCol, i); + psFree(multipleShifts); + psFree(table); + return false; + } + } + float x = psMetadataLookupS32(&mdok, row, xCol); // Shift in x + if (!mdok) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find column %s in row %d of shifts table\n", + xCol, i); + psFree(multipleShifts); + psFree(table); + return false; + } + float y = psMetadataLookupF32(&mdok, row, yCol); // Shift in y + if (!mdok) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find column %s in row %d of shifts table\n", + yCol, i); + psFree(multipleShifts); + psFree(table); + return false; + } + float t = psMetadataLookupF32(&mdok, row, tCol); // Time of shift + if (!mdok) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find column %s in row %d of shifts table\n", + tCol, i); + psFree(multipleShifts); + psFree(table); + return false; + } + + pmShifts *shifts = NULL; // Shifts for the cell of interest + switch (phuLevel) { + case PM_FPA_LEVEL_FPA: { + psHash *cells = psHashLookup(multipleShifts, chipName); // Hash of cells + if (!cells) { + cells = psHashAlloc(numCells); + psHashAdd(multipleShifts, chipName, cells); + psFree(cells); // Drop reference + } + shifts = cellVectors(cells, cellName, tRel, xyRel); + break; + } + case PM_FPA_LEVEL_CHIP: + shifts = cellVectors(multipleShifts, cellName, tRel, xyRel); + break; + case PM_FPA_LEVEL_CELL: + shifts = singleShifts; + break; + default: + psAbort("Should never get here.\n"); + } + + // Add the shift + psVectorExtend(shifts->x, 1, SHIFTS_BUFFER); + psVectorExtend(shifts->y, 1, SHIFTS_BUFFER); + psVectorExtend(shifts->t, 1, SHIFTS_BUFFER); + shifts->x->data.S32[shifts->num] = (int)x; + shifts->y->data.S32[shifts->num] = (int)y; + shifts->t->data.F32[shifts->num] = t; + shifts->num++; + } + psFree(table); + + // Put the kernels into their own cells + if (phuLevel == PM_FPA_LEVEL_CELL) { + // Only a single cell + psMetadataAddPtr(cell->analysis, PS_LIST_TAIL, PM_SHIFTS_TABLE_NAME, PS_DATA_KERNEL, + "Orthogonal transfer shifts", singleShifts); + psFree(singleShifts); + return true; + } else { + psList *names = psHashKeyList(multipleShifts); // List of hash keys (chip/cell names) + psListIterator *namesIter = psListIteratorAlloc(names, PS_LIST_HEAD, false); // Iterator for names + const char *name; // Name, from iteration + while ((name = psListGetAndIncrement(namesIter))) { + switch (phuLevel) { + case PM_FPA_LEVEL_FPA: { + int chipNum = pmFPAFindChip(fpa, name); // Number of chip of interest + if (chipNum < 0) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find chip %s", name); + psFree(namesIter); + psFree(names); + psFree(multipleShifts); + return false; + } + pmChip *chip = fpa->chips->data[chipNum]; // Chip of interest + + // Loop over component cells + psHash *cells = psHashLookup(multipleShifts, name); // Hash of cells + psList *cellNames = psHashKeyList(multipleShifts); // List of hash keys (cell names) + psListIterator *cellNamesIter = psListIteratorAlloc(cellNames, PS_LIST_HEAD, false); + const char *cellName; // Cell name, from iteration + while ((cellName = psListGetAndIncrement(cellNamesIter))) { + int cellNum = pmChipFindCell(chip, cellName); // Number of cell of interest + if (!cell) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find cell %s", cellName); + psFree(cellNamesIter); + psFree(cellNames); + psFree(namesIter); + psFree(names); + psFree(multipleShifts); + return false; + } + pmCell *cell = chip->cells->data[cellNum]; // Cell of interest + if (psMetadataLookup(cell->analysis, PM_SHIFTS_TABLE_NAME)) { + // Already has a shifts table, for some reason + psWarning("Chip %s, cell %s already has a shifts table --- overwriting\n", + name, cellName); + } + + pmShifts *vectors = psHashLookup(cells, cellName); // Shifts for the cell of interest + psMetadataAddPtr(cell->analysis, PS_LIST_TAIL, PM_SHIFTS_TABLE_NAME, + PS_DATA_KERNEL | PS_META_REPLACE, + "Orthogonal transfer shifts", vectors); + } + psFree(cellNamesIter); + psFree(cellNames); + break; + } + case PM_FPA_LEVEL_CHIP: { + int cellNum = pmChipFindCell(chip, name); // Number of cell of interest + if (!cell) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find cell %s", name); + psFree(namesIter); + psFree(names); + psFree(multipleShifts); + return false; + } + pmCell *cell = chip->cells->data[cellNum]; // Cell of interest + if (psMetadataLookup(cell->analysis, PM_SHIFTS_TABLE_NAME)) { + // Already has a shifts table, for some reason + psWarning("Cell %s already has a shifts table --- overwriting\n", name); + } + + pmShifts *vectors = psHashLookup(multipleShifts, name); // Shifts for this cell + psMetadataAddPtr(cell->analysis, PS_LIST_TAIL, PM_SHIFTS_TABLE_NAME, + PS_DATA_KERNEL | PS_META_REPLACE, + "Orthogonal transfer shifts", vectors); + break; + } + default: + psAbort("Should never get here.\n"); + } + } + psFree(namesIter); + psFree(names); + psFree(multipleShifts); + } + + // Go back to the original position in the FITS file + return psFitsMoveExtNum(fits, origExt, false); +} + + +// Generate a kernel and stuff it on the cell metadata +bool pmShiftsKernel(const pmCell *cell // Cell to which the shifts belong + ) +{ + PS_ASSERT_PTR(cell, false); + + bool mdok; // Status of MD lookup + pmShifts *shifts = psMetadataLookupPtr(&mdok, cell->analysis, PM_SHIFTS_TABLE_NAME); + if (!shifts) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find shifts table for cell.\n"); + return false; + } + + psKernel *kernel = psKernelGenerate(shifts->t, shifts->x, shifts->y, + shifts->tRelative, shifts->xyRelative); // Shift kernel + if (!kernel) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate kernel from OT shifts"); + return false; + } + psMetadataAddPtr(cell->analysis, PS_LIST_TAIL, PM_SHIFTS_KERNEL_NAME, PS_DATA_KERNEL, + "Orthogonal transfer kernel, calculated from shifts", kernel); + psFree(kernel); + + return true; +} + +bool pmShiftsConvolve(pmReadout *detrend, const pmCell *source, psMaskType maskVal) +{ + PS_ASSERT_PTR(detrend, false); + PS_ASSERT_PTR(source, false); + + bool mdok; // Status of MD lookup + psKernel *kernel = psMetadataLookupPtr(&mdok, source->analysis, PM_SHIFTS_KERNEL_NAME); + if (!kernel) { + // Maybe they just forgot to generate the kernel with pmShiftsKernel + if (psMetadataLookup(source->analysis, PM_SHIFTS_TABLE_NAME)) { + if (!pmShiftsKernel(source)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate shifts kernel."); + return false; + } + // Hopefully it's there now + kernel = psMetadataLookupPtr(&mdok, source->analysis, PM_SHIFTS_KERNEL_NAME); + if (!kernel) { + psError(PS_ERR_UNKNOWN, false, "Unable to find shifts kernel."); + return false; + } + } else { + psError(PS_ERR_UNKNOWN, false, "Unable to find shifts kernel or shifts table."); + return false; + } + } + + if (detrend->image) { +#if 1 + // Always do direct convolution (no fuss with edge effects) + psImage *convolved = psImageConvolveDirect(NULL, detrend->image, kernel); +#else + // Kernel size-dependent convolution --- if it's big, use the FFT + int xSize = kernel->xMax - kernel->xMin; // Kernel size in x + int ySize = kernel->yMax - kernel->yMin; // Kernel size in y + psImage *convolved; + if (xSize * ySize < FFT_SIZE * FFT_SIZE) { + convolved = psImageConvolveDirect(NULL, detrend->image, kernel); + } else { + // This is a little dodgy --- making choices about parameters without the user's input + psStats *stats = psImageStats(PS_STAT_ROBUST_MEDIAN); + stats->nSubsample = 10000; + psImageBackground(stats, detrend->image, detrend->mask, maskVal, NULL); + convolved = psImageConvolveFFT(detrend->image, kernel, stats->robustMedian); + } +#endif + if (!convolved) { + psError(PS_ERR_UNKNOWN, false, "Unable to convolve detrend image."); + return false; + } + psFree(detrend->image); + detrend->image = convolved; + } + + // Purposely ignoring the weight map --- don't care about the weight map for a detrend image + + if (maskVal && detrend->mask && !psImageConvolveMaskDirect(detrend->mask, detrend->mask, maskVal, 0, + kernel->xMin, kernel->xMax, + kernel->yMin, kernel->yMax)) { + psError(PS_ERR_UNKNOWN, false, "Unable to convolve detrend mask."); + return false; + } + + return true; +} Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmShifts.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmShifts.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmShifts.h (revision 22293) @@ -0,0 +1,51 @@ +#ifndef PM_SHIFTS_H +#define PM_SHIFTS_H + +#define PM_SHIFTS_TABLE_NAME "SHIFTS.TABLE" ///< Name for table on the analysis metadata +#define PM_SHIFTS_KERNEL_NAME "SHIFTS.KERNEL" ///< Name for kernel on the analysis metadata + +/// Shifts due to orthogonal transfer +typedef struct { + psVector *x; ///< Shifts in x + psVector *y; ///< Shifts in y + psVector *t; ///< Times of shifts + long num; ///< Number of values + bool tRelative; ///< Are the time values relative (durations)? + bool xyRelative; ///< Are the shift (x,y) values relative to the previous position? +} pmShifts; + +/// Allocator for pmShifts +pmShifts *pmShiftsAlloc(bool tRel, ///< Are the time values relative (durations)? + bool xyRel ///< Are the shift (x,y) values relative to the previous position? + ); + +/// Read orthogonal transfer shifts table for a cell +/// +/// Given a cell, this function searches for the orthogonal transfer shifts for this cell in the supplied FITS +/// file. The FITS extension containing the shifts table is specified by the SHIFTS keyword within the FILE +/// information in the camera format. If the extension is found, the shifts table is read and translated into +/// kernels which are placed in the analysis metadata of each cell. Note that this operation is performed on +/// all cells within the FITS file (or at least, those contained within the shifts table), not just the cell +/// provided --- if we have to read the whole table, we may as well translate the whole lot. If a kernel is +/// already present in the cell analysis metadata, the function returns true without doing any work. +bool pmShiftsRead(const pmCell *cell, ///< Cell for which to search for shifts + psFits *fits ///< FITS file in which to search for OT shifts extension + ); + + +/// Generate a kernel for the cell from the orthogonal transfer shifts +/// +/// The kernel is saved in the analysis metadata +bool pmShiftsKernel(const pmCell *cell ///< Cell for which to generate kernel + ); + +/// Convolve a detrend with the appropriate orthogonal transfer convolution kernel from a science exposure. +/// +/// The kernel is generated (with pmShiftsKernel) if required. The image and mask are convolved with the +/// kernel (specified maskVal is smeared). The weight map is not convolved. +bool pmShiftsConvolve(pmReadout *detrend, ///< Detrend readout to convolve + const pmCell *source, ///< Science exposure, containing a shifts kernel + psMaskType maskVal ///< Mask value to smear + ); + +#endif Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmShutterCorrection.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmShutterCorrection.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmShutterCorrection.c (revision 22293) @@ -0,0 +1,992 @@ +#if HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmHDUUtils.h" +#include "psVectorBracket.h" +#include "pmConceptsAverage.h" +#include "pmReadoutStack.h" + +#include "pmShutterCorrection.h" + +/// Measure shutter correction: +/// +/// input : collection of shutter correction exposures (pre-processed) +/// output : a shutter correction image +/// +/// The measurement could be performed on any focal-plane unit at a time. for GPC, the obvious scale is to +/// measure the effect on the entire focal plane at once, with a single reference point in the field. this is +/// a little more complex than just measuring the effect for a single 2D image array. the reference point and +/// the detailed analysis points need to be defined for the entire hierarchy rather than just as coordinate +/// pairs or regions. a pmFPAview would be a natural element with which to define these points, but at the +/// moment, the pmFPAview structure defines a band in the CCD, not a coordinate. An option is to instead +/// specify the reference locations as a pmFPAview coupled with a psRegion, though we need to be careful not +/// to over-specify the pixels (ie, conflict between pmFPAview and psRegion). +/// +/// At each point in an image with exposure time T, we measure f(k;T) = F(k;T) / F(0;T) where k is the +/// coordinate of the point of interest, 0 is the reference coordinate, and F(k;T) is the measured number of +/// counts at the point of interest in this image. given a collection of f(k;T) values, we need to determine +/// the model f(k;T) = A(k) (T + dTk) / (T + dTo) where dTk is the shutter error at the given position, dTo is +/// the shutter error at the reference position, and A(k) is the scaling factor for the given position. +/// +/// The process for generating a shutter correction is as follows: +/// - for each image +/// -- measure the reference point counts +/// - for each analysis region: +/// -- measure shutter parameters (dTo, dTk, A): +/// --- for each image: +/// ---- measure counts at the region +/// ---- divide by the reference counts +/// --- linear extrapolation to find f(inf) = A(k) +/// --- linear extrapolation to find f(0) = A(k) dTk / dTo +/// --- linear interpolation to find coordinate where f(dTo) = A (1 + dTk/dTo) / 2 +/// --- non-linear fit of T, f(T) to f(k;T) = A(k) (T + dTk) / (T + dTo) +/// - use the collection of dTo values to choose a best value for dTo (median) +/// - for each image pixel +/// -- divide by the reference counts +/// -- generate the vectors T, f(T) +/// -- linear fit of T, f(T) to f(k;T) = A(k) (T + dTk) / (T + dTo) using dTo above +/// -- save dTk, A(k) in output image pixels +/// -- apply dTk, A(k) to measure residual images +/// -- generate residual FITS/JPEG images + + +#define MEASURE_SAMPLES 4 // Number of samples to make over the image. This should only be + // changed with great caution, since assumptions on its value are in + // the code (see pmShutterCorrectionDataAlloc). + + +static void pmShutterCorrectionFree(pmShutterCorrection *pars) +{ + // Nothing to free + return; +} + +pmShutterCorrection *pmShutterCorrectionAlloc() +{ + pmShutterCorrection *corr = (pmShutterCorrection*)psAlloc(sizeof(pmShutterCorrection)); + psMemSetDeallocator(corr, (psFreeFunc)pmShutterCorrectionFree); + + corr->scale = 0.0; + corr->offset = 0.0; + corr->offref = 0.0; + + return corr; +} + +pmShutterCorrection *pmShutterCorrectionGuess(const psVector *exptime, const psVector *counts) +{ + // NOTE: vectors must be sorted on input. It is expensive to sort or check this here, but + // it is easy to arrange by sorting the images before generating these vectors. + + PS_ASSERT_VECTOR_NON_NULL(exptime, NULL); + PS_ASSERT_VECTOR_NON_NULL(counts, NULL); + PS_ASSERT_VECTOR_TYPE(exptime, PS_TYPE_F32, NULL); + PS_ASSERT_VECTOR_TYPE(counts, PS_TYPE_F32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(exptime, counts, NULL); + if (exptime->n <= 2) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Require more than 2 exposures to guess shutter correction.\n"); + return NULL; + } + + long N = exptime->n; // Number of exposures + + // use interpolation to guess shutter correction parameters given a set of exposures times and normalized + // counts (divided by the reference counts for each image) + + pmShutterCorrection *corr = pmShutterCorrectionAlloc(); // Shutter correction, to be returned + psPolynomial1D *line = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); // Straight line, for extrapolation + + // choose the highest exptime point as the guess for the scale: + // XXX we could examine the top 2 or 3 values and decide if we + // extended exptime enough or median clip. + corr->scale = counts->data.F32[N-1]; + + // fit a line to the lowest three points and extrapolate to 0.0 + psVector *tmpX = psVectorAlloc(2, PS_TYPE_F32); + psVector *tmpY = psVectorAlloc(2, PS_TYPE_F32); + + long index; + for (index = 0; !isfinite(exptime->data.F32[index]) && index < N - 1; index++) + ; // Iterate only + if (index == N - 1) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Not enough good values to guess shutter correction.\n"); + goto GUESS_ERROR; + } + tmpX->data.F32[0] = exptime->data.F32[index]; + tmpY->data.F32[0] = counts->data.F32[index]; + + for (index++; + (!isfinite(exptime->data.F32[index]) || exptime->data.F32[index] == exptime->data.F32[0]) && + index < N; index++) + ; // Iterate only + if (index == N) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Exposure times are all identical --- cannot guess shutter correction.\n"); + goto GUESS_ERROR; + } + tmpY->data.F32[1] = counts->data.F32[index]; + tmpX->data.F32[1] = exptime->data.F32[index]; + + // fit a line and extrapolate the fit to 0.0 + if (!psVectorFitPolynomial1D(line, NULL, 0, tmpY, NULL, tmpX)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to fit for the time offset.\n"); + goto GUESS_ERROR; + } + float ratio = psPolynomial1DEval(line, 0.0) / corr->scale; + + // XXX we need a sanity check: + // if the mean value of the three points is higher than corr->scale, + // then the slope should be negative. + // if the mean value of the three points is lower than corr->scale, + // then the slope should be positive. + + // find two points bracketing the value counts = A (1 + dTk/dTo) / 2 = corr->scale (1 + ratio) / 2 + float value = corr->scale * (1 + ratio) / 2.0; + + int Np; // Index of the value above (positive side) + if (ratio < 1.0) { + Np = psVectorBracket(counts, value, true); + } else { + Np = psVectorBracketDescend(counts, value, true); + } + int Nm = (Np == 0) ? 1 : Np - 1; // Index of the value below (negative side) + + tmpX->data.F32[0] = counts->data.F32[Nm]; + tmpX->data.F32[1] = counts->data.F32[Np]; + tmpY->data.F32[0] = exptime->data.F32[Nm]; + tmpY->data.F32[1] = exptime->data.F32[Np]; + + // fit a line and extrapolate the fit to counts = A (1 + dTk/dTo) : exptime = dTo + if (!psVectorFitPolynomial1D (line, NULL, 0, tmpY, NULL, tmpX)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to fit for the reference offset.\n"); + goto GUESS_ERROR; + } + corr->offref = psPolynomial1DEval(line, value); + corr->offset = ratio * corr->offref; + + psFree(line); + psFree(tmpX); + psFree(tmpY); + + return corr; + +GUESS_ERROR: + psFree(tmpX); + psFree(tmpY); + psFree(line); + psFree(corr); + return NULL; +} + +// linear fit to the counts and exptime, given a value for offref +pmShutterCorrection *pmShutterCorrectionLinFit(const psVector *exptime, + const psVector *counts, + const psVector *cntError, + const psVector *mask, + float offref, + int nIter, + float rej, + psMaskType maskVal + ) +{ + PS_ASSERT_VECTOR_NON_NULL(exptime, NULL); + PS_ASSERT_VECTOR_NON_NULL(counts, NULL); + PS_ASSERT_VECTOR_TYPE(exptime, PS_TYPE_F32, NULL); + PS_ASSERT_VECTOR_TYPE(counts, PS_TYPE_F32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(exptime, counts, NULL); + if (exptime->n <= 2) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Require more than 2 exposures to guess shutter correction.\n"); + return NULL; + } + if (cntError) { + PS_ASSERT_VECTOR_TYPE(cntError, PS_TYPE_F32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(counts, cntError, NULL); + } + PS_ASSERT_FLOAT_LARGER_THAN(offref, 0.0, NULL); + + // this step is identical for all pixels: do it once and save? + psVector *x = psVectorAlloc(exptime->n, PS_TYPE_F32); + psVector *y = psVectorAlloc(exptime->n, PS_TYPE_F32); + + for (long i = 0; i < exptime->n; i++) { + // Should be safe (if expensive) to stick NaNs in --- the fitter deals with them + float value = 1.0 / (exptime->data.F32[i] + offref); + x->data.F32[i] = exptime->data.F32[i] * value; + y->data.F32[i] = value; + } + + psPolynomial2D *line = psPolynomial2DAlloc (PS_POLYNOMIAL_ORD, 1, 1); + + // mask out the terms we will not fit + line->coeffMask[0][0] = PS_POLY_MASK_SET; + line->coeffMask[1][1] = PS_POLY_MASK_SET; + line->coeff[0][0] = 0; + line->coeff[1][1] = 0; + + // the stats structure determines how the clipping statistic is measured + // too few points to use the robust analysis method + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV); + stats->clipSigma = rej; + stats->clipIter = nIter; + + if (!psVectorClipFitPolynomial2D(line, stats, mask, maskVal, counts, cntError, x, y)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to fit shutter correction.\n"); + psFree(stats); + psFree(x); + psFree(y); + psFree(line); + return NULL; + } + psFree(stats); + + pmShutterCorrection *corr = pmShutterCorrectionAlloc(); + corr->offref = offref; + corr->scale = line->coeff[1][0]; + corr->offset = line->coeff[0][1] / line->coeff[1][0]; + + psFree(x); + psFree(y); + psFree(line); + + return corr; +} + +static psF32 pmShutterCorrectionModel(psVector *deriv, + const psVector *params, + const psVector *x) +{ + // This is in a tight loop, so we won't assert here. + + psF32 A = params->data.F32[0]; + psF32 p = x->data.F32[0] + params->data.F32[1]; + psF32 q = 1.0 / (x->data.F32[0] + params->data.F32[2]); + psF32 f = A * p * q; + + if (deriv) { + deriv->data.F32[0] = p * q; + deriv->data.F32[1] = A * q; + deriv->data.F32[2] = - f * q; + } + return f; +} + +// non-linear fit to the counts and exptime, given a guess for the three parameters +pmShutterCorrection *pmShutterCorrectionFullFit(const psVector *exptime, + const psVector *counts, + const psVector *cntError, + const pmShutterCorrection *guess + ) +{ + PS_ASSERT_VECTOR_NON_NULL(exptime, NULL); + PS_ASSERT_VECTOR_NON_NULL(counts, NULL); + PS_ASSERT_VECTOR_TYPE(exptime, PS_TYPE_F32, NULL); + PS_ASSERT_VECTOR_TYPE(counts, PS_TYPE_F32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(exptime, counts, NULL); + if (exptime->n <= 2) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Require more than 2 exposures to guess shutter correction.\n"); + return NULL; + } + if (cntError) { + PS_ASSERT_VECTOR_TYPE(cntError, PS_TYPE_F32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(counts, cntError, NULL); + } + PS_ASSERT_PTR_NON_NULL(guess, NULL); + + psMinimization *minInfo = psMinimizationAlloc(15, 0.1); // Minimization information + + psVector *params = psVectorAlloc (3, PS_TYPE_F32); // Fitting parameters + params->data.F32[0] = guess->scale; + params->data.F32[1] = guess->offset; + params->data.F32[2] = guess->offref; + + // XXX for the moment, don't set any constraints + // psMinConstraint *constraint = psMinConstraintAlloc(); + // constrain->checkLimits = pmShutterParamLimits; + // constrain->paramMask = NULL; + psMinConstraint *constraint = NULL; // Constraints on the minimization + + // XXX ignore covariance matrix for the moment + // psImage *covar = psImageAlloc (params->n, params->n, PS_TYPE_F64); + psImage *covar = NULL; // Covariance matrix + + // construct the coordinate and value entries (y is counts) + psArray *x = psArrayAlloc(exptime->n); // Coordinates + + for (long i = 0; i < exptime->n; i++) { + psVector *coord = psVectorAlloc(1, PS_TYPE_F32); + coord->data.F32[0] = exptime->data.F32[i]; + x->data[i] = coord; + } + + if (!psMinimizeLMChi2(minInfo, covar, params, constraint, x, counts, cntError, pmShutterCorrectionModel)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to fit for shutter correction.\n"); + psFree(x); + psFree(minInfo); + psFree(params); + return NULL; + } + + pmShutterCorrection *corr = pmShutterCorrectionAlloc(); // Shutter correction + corr->scale = params->data.F32[0]; + corr->offset = params->data.F32[1]; + corr->offref = params->data.F32[2]; + + psFree(minInfo); + psFree(params); + psFree(x); + + return corr; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +bool pmShutterCorrectionMeasure(pmReadout *output, const psArray *readouts, int size, psStatsOptions meanStat, + psStatsOptions stdevStat, int nIter, float rej, psMaskType maskVal) +{ + PS_ASSERT_ARRAY_NON_NULL(readouts, NULL); + PS_ASSERT_ARRAY_NON_EMPTY(readouts, NULL); + PS_ASSERT_INT_POSITIVE(size, NULL); + + long num = readouts->n; // Number of readouts + PS_ASSERT_INT_POSITIVE(nIter, NULL); + PS_ASSERT_FLOAT_LARGER_THAN(rej, 0.0, NULL); + + psArray *images = psArrayAlloc(num);// Array of images + psArray *masks = NULL; // Array of masks + psArray *weights = NULL; // Array of weights + psVector *exptimes = psVectorAlloc(num, PS_TYPE_F32); // Vector of exposure times + + { + pmReadout *readout = readouts->data[0]; // Representative readout + if (readout->mask) + { + masks = psArrayAlloc(num); + } + if (readout->weight) + { + weights = psArrayAlloc(num); + } + } + + // Check input sizes, generate first-pass statistics + psRegion refRegion; // Reference region + psVector *refs = psVectorAlloc(num, PS_TYPE_F32); // Reference measurements + psVectorInit(refs, 0); + psArray *regions = psArrayAlloc(MEASURE_SAMPLES); // Array of sample regions, made on each image + psImage *samplesMean = psImageAlloc(num, MEASURE_SAMPLES, PS_TYPE_F32); // Measurements for each file + psImage *samplesStdev = psImageAlloc(num, MEASURE_SAMPLES, PS_TYPE_F32); // Errors for each file + psStats *stats = psStatsAlloc(meanStat | stdevStat); + int numRows = 0, numCols = 0; // Size of images + for (long i = 0; i < images->n; i++) { + pmReadout *readout = readouts->data[i]; // Readout of interest + if (!readout) { + continue; + } + + bool mdok; // Status of MD lookup + float exptime = psMetadataLookupF32(&mdok, readout->parent->concepts, "CELL.EXPOSURE"); // Exp. time + if (!mdok || !isfinite(exptime)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure time for readout %ld is not set.\n", i); + goto MEASURE_ERROR; + } + exptimes->data.F32[i] = exptime; + + psImage *image = readout->image; // Image of interest + if (!image) { + continue; + } + images->data[i] = psMemIncrRefCounter(image); + if (image->type.type != PS_TYPE_F32) { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Bad type for image: %x\n", image->type.type); + goto MEASURE_ERROR; + } + if (numRows == 0 || numCols == 0) { + numRows = image->numRows; + numCols = image->numCols; + // Set up the sample regions + refRegion = psRegionForSquare(0.5 * numCols, 0.5 * numRows, size); + for (int j = 0; j < MEASURE_SAMPLES; j++) { + int x = (j % 2) ? size : image->numCols - size; + int y = (j > 1) ? size : image->numRows - size; + psRegion region = psRegionForSquare(x, y, size); + region = psRegionForImage(image, region); + regions->data[j] = psRegionAlloc(region.x0, region.x1, region.y0, region.y1); + } + } else if (numRows != image->numRows || numCols != image->numCols) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Image sizes don't match: %dx%d vs %dx%d\n", image->numCols, image->numRows, + numCols, numRows); + goto MEASURE_ERROR; + } + psImage *mask = readout->mask; // Mask of interest + if (mask) { + if (!masks) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Not all readouts have masks.\n"); + goto MEASURE_ERROR; + } + masks->data[i] = psMemIncrRefCounter(mask); + + if (mask->type.type != PS_TYPE_U8) { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Bad type for mask: %x\n", mask->type.type); + goto MEASURE_ERROR; + } + if (mask->numRows != numRows || mask->numCols != numCols) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Mask sizes don't match: %dx%d vs %dx%d\n", mask->numCols, mask->numRows, + numCols, numRows); + goto MEASURE_ERROR; + } + } else if (masks) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Not all readouts have masks.\n"); + goto MEASURE_ERROR; + } + + psImage *weight = readout->weight; // Weight map of interest + if (weight) { + if (!weights) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Not all readouts have weights.\n"); + goto MEASURE_ERROR; + } + weights->data[i] = psMemIncrRefCounter(weight); + + if (weight->type.type != PS_TYPE_F32) { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Bad type for weights: %x\n", weight->type.type); + goto MEASURE_ERROR; + } + if (weight->numRows != numRows || weight->numCols != numCols) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Weight sizes don't match: %dx%d vs %dx%d\n", weight->numCols, weight->numRows, + numCols, numRows); + goto MEASURE_ERROR; + } + } else if (weights) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Not all readouts have weights.\n"); + goto MEASURE_ERROR; + } + + + // Measure statistics + if (!psImageStats(stats, image, mask, maskVal)) { + psWarning("Unable to measure reference statistics.\n"); + } + refs->data.F32[i] = psStatsGetValue(stats, meanStat); + psTrace("psModules.detrend", 3, "Reference value for image %ld = %f\n", i, refs->data.F32[i]); + if (refs->data.F32[i] <= 0.0) { + psError(PS_ERR_UNKNOWN, true, "Measured non-positive reference value.\n"); + goto MEASURE_ERROR; + } + refs->data.F32[i] = 1.0 / refs->data.F32[i]; + for (int j = 0; j < MEASURE_SAMPLES; j++) { + psRegion *region = regions->data[j]; // Region of interest + psImage *subImage = psImageSubset(image, *region); // Sub-image + psImage *subMask = NULL; + if (mask) { + subMask = psImageSubset(mask, *region); + } + if (!psImageStats(stats, subImage, subMask, maskVal)) { + psString regionString = psRegionToString(*region); + psWarning("Unable to measure sample statistics at %s in image %ld.\n", + regionString, i); + psFree(regionString); + } + psFree(subImage); + samplesMean->data.F32[j][i] = psStatsGetValue(stats, meanStat) * refs->data.F32[i]; + samplesStdev->data.F32[j][i] = psStatsGetValue(stats, stdevStat) * refs->data.F32[i]; + psTrace("psModules.detrend", 5, "Image %ld, sample %d: %f +/- %f\n", i, j, + samplesMean->data.F32[j][i], samplesStdev->data.F32[j][i]); + } + } + psFree(regions); + psFree(stats); + + float meanRef = 0.0; // Mean reference offset + int numGood = 0; // Number of good measurements + psVector *counts = psVectorAlloc(num, PS_TYPE_F32); // Mean for each image + psVector *errors = psVectorAlloc(num, PS_TYPE_F32); // Stdev for each image + for (int i = 0; i < MEASURE_SAMPLES; i++) { + counts = psImageRow(counts, samplesMean, i); + errors = psImageRow(errors, samplesStdev, i); + pmShutterCorrection *guess = pmShutterCorrectionGuess(exptimes, counts); // Guess at correction + pmShutterCorrection *corr = pmShutterCorrectionFullFit(exptimes, counts, errors, guess); // Correct'n + if (!corr) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to measure shutter reference correction.\n"); + psFree(guess); + psFree(counts); + psFree(errors); + goto MEASURE_ERROR; + } + psTrace("psModules.detrend", 5, "Sample reference value: %f\n", corr->offref); + if (isfinite(corr->offref)) { + meanRef += corr->offref; + numGood++; + } + psFree(corr); + psFree(guess); + } + psFree(samplesMean); + psFree(samplesStdev); + + if (numGood == 0) { + psError(PS_ERR_UNKNOWN, true, "Unable to measure mean reference offset.\n"); + psFree(counts); + psFree(errors); + goto MEASURE_ERROR; + } + meanRef /= (float)numGood; + psTrace("psModules.detrend", 3, "Mean reference value: %f\n", meanRef); + + // Check the weights + if (weights && nIter > 1) { + for (int i = 0; i < weights->n && nIter > 1; i++) { + psImage *weight = weights->data[i]; // Weight image + if (!weight) { + // We don't have weights, so no realistic errors: turn off iteration + if (nIter > 0) { + psWarning("Not all images have weights --- turning iteration off.\n"); + } + nIter = 1; + } + } + } + + psImage *shutter = psImageAlloc(numCols, numRows, PS_TYPE_F32); // Shutter correction image + psImage *pattern = psImageAlloc(numCols, numRows, PS_TYPE_F32); // Illumination pattern + psVector *mask = psVectorAlloc(num, PS_TYPE_U8); // Mask for each image + psVectorInit(mask, 0); + psTrace("psModules.detrend", 2, "Performing linear fit on individual pixels...\n"); + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + for (int i = 0; i < num; i++) { + psImage *image = images->data[i]; // Image of interest + counts->data.F32[i] = image->data.F32[y][x] * refs->data.F32[i]; + psImage *maskImage; // Mask image + if (masks && (maskImage = masks->data[i])) { + mask->data.U8[i] = maskImage->data.U8[y][x]; + } + psImage *weight; // Weight image + if (weights && (weight = weights->data[i])) { + errors->data.F32[i] = sqrtf(weight->data.F32[y][x]) * refs->data.F32[i]; + } else { + errors->data.F32[i] = sqrtf(image->data.F32[y][x]) * refs->data.F32[i]; + } + } + + pmShutterCorrection *corr = pmShutterCorrectionLinFit(exptimes, counts, errors, mask, meanRef, + nIter, rej, maskVal); + shutter->data.F32[y][x] = corr->offset; + pattern->data.F32[y][x] = corr->scale; + psFree(corr); + } + } + psFree(mask); + psFree(counts); + psFree(errors); + psFree(refs); + + if (psTraceGetLevel("psModules.detrend") > 5) { + psFits *fits = psFitsOpen("pattern.fits", "w"); + psFitsWriteImage(fits, NULL, pattern, 0, NULL); + psFitsClose(fits); + } + psFree(pattern); + + output->image = shutter; + + // Update the "concepts" + psList *inputCells = psListAlloc(NULL); // List of cells + for (long i = 0; i < readouts->n; i++) { + pmReadout *readout = readouts->data[i]; // Readout of interest + psListAdd(inputCells, PS_LIST_TAIL, readout->parent); + } + bool success = pmConceptsAverageCells(output->parent, inputCells, NULL, NULL, true); + psFree(inputCells); + + // Correct the exposure times --- they don't make sense any more. + psMetadataItem *item = psMetadataLookup(output->parent->concepts, "CELL.EXPOSURE"); + item->data.F32 = NAN; + item = psMetadataLookup(output->parent->concepts, "CELL.DARKTIME"); + item->data.F32 = NAN; + + return success; + + +MEASURE_ERROR: + // Clean up after error + psFree(exptimes); + psFree(images); + psFree(masks); + psFree(weights); + psFree(refs); + psFree(regions); + psFree(stats); + psFree(samplesMean); + psFree(samplesStdev); + return false; +} + + +bool pmShutterCorrectionApply(pmReadout *readout, const pmReadout *shutter) +{ + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_PTR_NON_NULL(shutter, false); + PS_ASSERT_IMAGE_NON_NULL(readout->image, false); + PS_ASSERT_IMAGE_NON_NULL(shutter->image, false); + PS_ASSERT_IMAGE_TYPE(readout->image, PS_TYPE_F32, false); + PS_ASSERT_IMAGE_TYPE(shutter->image, PS_TYPE_F32, false); + + psRegion region = psRegionSet(readout->col0, readout->col0 + readout->image->numCols, + readout->row0, readout->row0 + readout->image->numRows); // Detector region + + pmCell *cell = readout->parent; // Parent cell + if (!cell) { + psError(PS_ERR_BAD_PARAMETER_NULL, true, + "Parent cell is NULL --- unable to determine exposure time.\n"); + return false; + } + float exptime = psMetadataLookupF32(NULL, cell->concepts, "CELL.EXPOSURE"); // Exposure time + if (!isfinite(exptime)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Bad exposure time: %f.\n", exptime); + return false; + } + + pmHDU *hdu = pmHDUFromCell(cell);// HDU of interest + + psVector *md5 = psImageMD5(shutter->image); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psStringPrepend(&md5string, "Shutter image MD5: "); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, md5string, ""); + psFree(md5string); + + + psImage *shutterImage = psImageSubset(shutter->image, region); // Subimage with shutter + if (!shutterImage) { + psString regionString = psRegionToString(region); + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Size mismatch: %s vs %dx%d\n", + regionString, shutter->image->numCols, shutter->image->numRows); + psFree(regionString); + psFree(shutterImage); + return false; + } + psImage *image = readout->image; // Image to correct + + if (exptime <= 0.0) { + // In the extreme case that we have exptime <= 0.0, we correct the image to + // counts-per-second, rather than counts in the nominal exposure time + for (int y = 0; y < image->numRows; y++) { + for (int x = 0; x < image->numCols; x++) { + image->data.F32[y][x] *= 1.0 / (exptime + shutterImage->data.F32[y][x]); + } + } + psMetadataAddF32 (cell->concepts, PS_LIST_TAIL, "CELL.EXPOSURE", PS_META_REPLACE, "exposure time re-normalized to 1.0", 1.0); // Exposure time + psString line = NULL; + psStringAppend (&line, "extreme exposure time %f, re-normalized to 1.0", exptime); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, line, ""); + psFree (line); + } else { + for (int y = 0; y < image->numRows; y++) { + for (int x = 0; x < image->numCols; x++) { + image->data.F32[y][x] *= exptime / (exptime + shutterImage->data.F32[y][x]); + } + } + } + psFree(shutterImage); + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now, used for reporting + psString timeString = psTimeToISO(time); // String with time + psFree(time); + psStringPrepend(&timeString, "Shutter correction completed at "); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + timeString, ""); + psFree(timeString); + + + return true; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + +#define IMAGES_BUFFER 10 // Allocate space for this many images at a time + +static void shutterCorrectionDataFree(pmShutterCorrectionData *data) +{ + psFree(data->regions); + psFree(data->mean); + psFree(data->stdev); + + psFree(data->exptimes); + psFree(data->refs); + + return; +} + +pmShutterCorrectionData *pmShutterCorrectionDataAlloc(int numCols, int numRows, int size) +{ + pmShutterCorrectionData *data = psAlloc(sizeof(pmShutterCorrectionData)); + psMemSetDeallocator(data, (psFreeFunc)shutterCorrectionDataFree); + + data->num = 0; + data->numCols = 0; + data->numRows = 0; + + data->regions = psArrayAlloc(MEASURE_SAMPLES); + for (int j = 0; j < MEASURE_SAMPLES; j++) { + int x = (j % 2) ? size : numCols - size - 1; + int y = (j > 1) ? size : numRows - size - 1; + psRegion region = psRegionForSquare(x, y, size); + data->regions->data[j] = psRegionAlloc(region.x0, region.x1, region.y0, region.y1); + } + + data->mean = psArrayAlloc(MEASURE_SAMPLES); + data->stdev = psArrayAlloc(MEASURE_SAMPLES); + for (int i = 0; i < MEASURE_SAMPLES; i++) { + data->mean->data[i] = psVectorAllocEmpty(IMAGES_BUFFER, PS_TYPE_F32); + data->stdev->data[i] = psVectorAllocEmpty(IMAGES_BUFFER, PS_TYPE_F32); + } + + data->exptimes = psVectorAllocEmpty(IMAGES_BUFFER, PS_TYPE_F32); + data->refs = psVectorAllocEmpty(IMAGES_BUFFER, PS_TYPE_F32); + + return data; +} + +bool pmShutterCorrectionAddReadout(pmShutterCorrectionData *data, + const pmReadout *readout, ///< Readout to add + psStatsOptions meanStat, ///< Statistic to use for mean + psStatsOptions stdevStat, ///< Statistic to use for stdev + psMaskType maskVal, ///< Mask value + psRandom *rng ///< Random number generator + ) +{ + PS_ASSERT_PTR_NON_NULL(data, NULL); + PS_ASSERT_PTR_NON_NULL(readout, NULL); + PS_ASSERT_IMAGE_NON_NULL(readout->image, NULL); + PS_ASSERT_IMAGE_TYPE(readout->image, PS_TYPE_F32, NULL); + if (data->num == 0) { + data->numCols = readout->image->numCols; + data->numRows = readout->image->numRows; + } else { + PS_ASSERT_IMAGE_SIZE(readout->image, data->numCols, data->numRows, NULL); + } + if (readout->mask) { + PS_ASSERT_IMAGE_NON_NULL(readout->mask, NULL); + PS_ASSERT_IMAGE_TYPE(readout->mask, PS_TYPE_MASK, NULL); + PS_ASSERT_IMAGE_SIZE(readout->mask, data->numCols, data->numRows, NULL); + } + if (readout->weight) { + PS_ASSERT_IMAGE_NON_NULL(readout->weight, NULL); + PS_ASSERT_IMAGE_TYPE(readout->weight, PS_TYPE_F32, NULL); + PS_ASSERT_IMAGE_SIZE(readout->weight, data->numCols, data->numRows, NULL); + } + + // Add the exposure time + float exptime = psMetadataLookupF32(NULL, readout->parent->concepts, "CELL.EXPOSURE"); // Exp. time + if (!isfinite(exptime)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure time is not set."); + return false; + } + data->exptimes->data.F32[data->exptimes->n] = exptime; + data->exptimes = psVectorExtend(data->exptimes, IMAGES_BUFFER, 1); + + // Add the statistics + + // Add the reference value + psStats *stats = psStatsAlloc(meanStat | stdevStat); // Statistics to apply + if (!rng) { + rng = psRandomAlloc(PS_RANDOM_TAUS, 0); + } else { + psMemIncrRefCounter(rng); + } + if (!psImageBackground(stats, NULL, readout->image, readout->mask, maskVal, rng)) { + psError(PS_ERR_UNKNOWN, false, "Unable to measure reference statistics.\n"); + psFree(stats); + psFree(rng); + return false; + } + psFree(rng); + float refValue = psStatsGetValue(stats, meanStat); // Reference value + psTrace("psModules.detrend", 3, "Reference value for shutter image = %f\n", refValue); + if (refValue <= 0.0) { + psError(PS_ERR_UNKNOWN, true, "Measured non-positive reference value.\n"); + psFree(stats); + return false; + } + refValue = 1.0 / refValue; + data->refs->data.F32[data->refs->n] = refValue; + data->refs = psVectorExtend(data->refs, IMAGES_BUFFER, 1); + + // Add the region statistics + for (int j = 0; j < MEASURE_SAMPLES; j++) { + psRegion *region = data->regions->data[j]; // Region of interest + psRegion adjusted = *region; // Adjusted region, compensating for offsets + adjusted.x0 += readout->image->col0; + adjusted.x1 += readout->image->col0; + adjusted.y0 += readout->image->row0; + adjusted.y1 += readout->image->row0; + psImage *subImage = psImageSubset(readout->image, adjusted); // Sub-image + psImage *subMask = NULL; // Sub-image of mask + if (readout->mask) { + subMask = psImageSubset(readout->mask, adjusted); + } + if (!psImageStats(stats, subImage, subMask, maskVal)) { + psString regionString = psRegionToString(adjusted); + psWarning("Unable to measure sample statistics at %s in image.\n", + regionString); + psFree(regionString); + } + psFree(subImage); + psFree(subMask); + + psVector *mean = data->mean->data[j]; // Vector of means for this region + psVector *stdev = data->stdev->data[j]; // Vector of standard deviations for this region + + mean->data.F32[mean->n] = psStatsGetValue(stats, meanStat) * refValue; + stdev->data.F32[stdev->n] = psStatsGetValue(stats, stdevStat) * refValue; + data->mean->data[j] = psVectorExtend(mean, IMAGES_BUFFER, 1); + data->stdev->data[j] = psVectorExtend(stdev, IMAGES_BUFFER, 1); + } + + data->num++; + + return true; +} + +float pmShutterCorrectionReference(const pmShutterCorrectionData *data) +{ + PS_ASSERT_PTR_NON_NULL(data, NAN); + PS_ASSERT_INT_POSITIVE(data->num, NAN); + + float meanRef = 0.0; // Mean reference offset + int numGood = 0; // Number of good measurements + for (int i = 0; i < MEASURE_SAMPLES; i++) { + psVector *counts = data->mean->data[i]; // Mean for each image + psVector *errors = data->stdev->data[i]; // Stdev for each image + pmShutterCorrection *guess = pmShutterCorrectionGuess(data->exptimes, counts); // Guess at correction + pmShutterCorrection *corr = pmShutterCorrectionFullFit(data->exptimes, counts, + errors, guess); // The actual correction + psFree(guess); + if (corr && isfinite(corr->offref)) { + psTrace("psModules.detrend", 5, "Sample reference value: %f\n", corr->offref); + meanRef += corr->offref; + numGood++; + } + psFree(corr); + } + + if (numGood == 0) { + psError(PS_ERR_UNKNOWN, true, "Unable to measure mean reference offset.\n"); + return false; + } + meanRef /= (float)numGood; + psTrace("psModules.detrend", 3, "Mean reference value: %f\n", meanRef); + return meanRef; +} + +bool pmShutterCorrectionGenerate(pmReadout *shutter, pmReadout *pattern, const psArray *inputs, + float reference, const pmShutterCorrectionData *data, + int nIter, float rej, psMaskType maskVal) +{ + PS_ASSERT_PTR_NON_NULL(shutter, false); + PS_ASSERT_ARRAY_NON_NULL(inputs, false); + PS_ASSERT_INT_EQUAL(data->num, inputs->n, false); + + int minInputCols, maxInputCols, minInputRows, maxInputRows; // Smallest and largest values to combine + int xSize, ySize; // Size of the output image + if (!pmReadoutStackValidate(&minInputCols, &maxInputCols, &minInputRows, &maxInputRows, &xSize, &ySize, + inputs)) { + psError(PS_ERR_UNKNOWN, false, "No valid input readouts."); + return false; + } + + pmReadoutUpdateSize(shutter, minInputCols, minInputRows, xSize, ySize, false, false, maskVal); + if (pattern) { + pmReadoutUpdateSize(pattern, minInputCols, minInputRows, xSize, ySize, false, false, maskVal); + } + + psImage *shutterImage = shutter->image; // Shutter correction image + psImage *patternImage; // Illumination pattern + if (pattern) { + patternImage = psMemIncrRefCounter(pattern->image); + } else { + patternImage = psImageAlloc(xSize, ySize, PS_TYPE_F32); + } + + int num = data->num; // Number of images + psVector *counts = psVectorAlloc(num, PS_TYPE_F32); // Counts in each image + psVector *errors = psVectorAlloc(num, PS_TYPE_F32); // Counts in each image + psVector *mask = psVectorAlloc(num, PS_TYPE_MASK); // Mask for each image + psTrace("psModules.detrend", 2, "Performing linear fit on individual pixels...\n"); + for (int i = minInputRows; i < maxInputRows; i++) { + int yOut = i - shutter->row0; // y position on output readout + for (int j = minInputCols; j < maxInputCols; j++) { + int xOut = j - shutter->col0; // x position on output readout + + psVectorInit(mask, 0); + for (int r = 0; r < num; r++) { + pmReadout *readout = inputs->data[r]; // Readout of interest + int yIn = i - readout->row0; // y position on input readout + int xIn = j - readout->col0; // x position on input readout + psImage *image = readout->image; // Image of interest + float ref = data->refs->data.F32[r]; // (Inverse) reference value + counts->data.F32[r] = image->data.F32[yIn][xIn] * ref; + if (readout->mask) { + mask->data.PS_TYPE_MASK_DATA[r] = readout->mask->data.PS_TYPE_MASK_DATA[yIn][xIn]; + } + if (readout->weight) { + errors->data.F32[r] = sqrtf(readout->weight->data.F32[yIn][xIn]) * ref; + } else { + errors->data.F32[r] = sqrtf(image->data.F32[yIn][xIn]) * ref; + } + } + + pmShutterCorrection *corr = pmShutterCorrectionLinFit(data->exptimes, counts, errors, mask, + reference, nIter, rej, maskVal); + shutterImage->data.F32[yOut][xOut] = corr->offset; + patternImage->data.F32[yOut][xOut] = corr->scale; + psFree(corr); + } + } + psFree(mask); + psFree(errors); + psFree(counts); + + // Update the "concepts" + psList *inputCells = psListAlloc(NULL); // List of cells + for (long i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + psListAdd(inputCells, PS_LIST_TAIL, readout->parent); + } + bool success = pmConceptsAverageCells(shutter->parent, inputCells, NULL, NULL, true); + psFree(inputCells); + + // Correct the exposure times --- they don't make sense any more. + psMetadataItem *item = psMetadataLookup(shutter->parent->concepts, "CELL.EXPOSURE"); + item->data.F32 = NAN; + item = psMetadataLookup(shutter->parent->concepts, "CELL.DARKTIME"); + item->data.F32 = NAN; + + shutter->data_exists = true; + shutter->parent->data_exists = true; + shutter->parent->parent->data_exists = true; + + if (pattern) { + pattern->data_exists = true; + pattern->parent->data_exists = true; + pattern->parent->parent->data_exists = true; + } + + return success; +} Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmShutterCorrection.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmShutterCorrection.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmShutterCorrection.h (revision 22293) @@ -0,0 +1,186 @@ +/* @file pmShutterCorrection.h + * @brief Functions to build and apply a shutter exposure-time correction. + * + * @author Eugene Magnier, IfA + * @author Paul Price, IfA + * + * @version $Revision: 1.14 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-06-19 03:40:48 $ + * Copyright 2006 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_SHUTTER_CORRECTION_H +#define PM_SHUTTER_CORRECTION_H + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +/* A mechanical shutter may not yield uniform exposure times as a function of + * position on the detector. The typical error consists of a constant + * exposure-time offset relative to the requested value, ie exposure time is + * T_o + dT(x,y). The exposure error, dT, may be measured with the following + * scheme. Obtain a set of exposures with different exposures times taken of + * the same flat-field source; the source must be spatially stable between the + * exposures, but need not have a stable amplitude. For an illuminating flux + * of intensity F(x,y) = F_o f(x,y), the signal recorded by any pixel in the + * detector is given by: S(t,x,y) = F_o(t) f(x,y) (T_o + dT(x,y)) where F_o is + * the F_o(t) is the (variable) overall intensity of the illuminating source + * and f(x,y) is the spatial illumination patter times the flat-field response. + * Choose a reference location in the image (eg, the detector center) and + * divide by the value of that region (ie, mean or median): + * + * s(t,x,y) = S(t,x,y) / S(t,0,0) + * s(t,x,y) = F_o(t) f(x,y) (T_o + dT(x,y)) / F_o(t) f(0,0) (T_o + dT(0,0)) + * s(t,x,y) = f(x,y) (T_o + dT(x,y)) / f(0,0) (T_o + dT(0,0)) + * + * we can absorb the term f(0,0) into f(x,y) as we have no motivation for the + * scale of f(x,y). For any single pixel, over the set of exposures, we thus + * need to solve for dT(x,y), dT(0,0), and f'(x,y) in the equation: + * s(t,x,y) = f'(x,y) (T_o + dT(x,y)) / (T_o + dT(0,0)) + * + * we avoid directly fitting these values as the process would be a non-linear + * least-squares problem for every pixel in the image, and thus very time + * consuming. There are linear options which may be used instead. + * First, as T_o goes to a large value, s() approaches the value of f'(x,y). + * Next, as T_o goes to a very small value, s() approaches the value of + * f'(x,y)*dT(x,y)/dT(0,0). Finally, when s() has the value of + * f'(x,y)*(1 + dT(x,y)/dT(0,0))/2, T_o has the value of dT(0,0). with data + * points covering a reasonable dynamic range, we can solve for these three + * values by interpolation and/or extrapolation. + * + * To take the strategy one step further, we could use the above recipe to + * obtain a guess for the three parameters and then apply non-linear fitting to + * solve more accurately for the parameters. If we limit this operation to a + * handful of positions in the image (user defined, but the obvious choice would + * be positions near the center, edges, and corners), then we may determine a + * good value for dT(0,0). Since there is only one dT(0,0) for the image, we + * can apply the resulting measurement to the rest of the pixels in the image. + * If dT(0,0) is not a free parameter, then the fitting process is linear in + * terms of dT(x,y) and f'(x,y) + */ + +/// Shutter correction parameters, applicable for a single pixel +typedef struct +{ + double scale; ///< The normalisation for an exposure, A(k) + double offset; ///< The time offset, dTk + double offref; ///< The reference time offset, dTo +} +pmShutterCorrection; + +/// Allocator for shutter correction parameters +pmShutterCorrection *pmShutterCorrectionAlloc(); + +/// Guess a shutter correction, based on plot of counts vs exposure time +/// +/// This function is used before doing the full non-linear fit, to get parameters close to the true. Assumes +/// exptime vector is sorted (ascending order; longest is last) prior to input. +pmShutterCorrection *pmShutterCorrectionGuess(const psVector *exptime, ///< Exposure times for each exposure + const psVector *counts ///< Counts for each exposure + ); + +/// Generate shutter correction based on a linear fit +/// +/// Performs a linear fit to counts as a function of exposure time, with the reference time offset fixed (so +/// that the system is linear). Performs iterative clipping, if nIter > 1. +pmShutterCorrection *pmShutterCorrectionLinFit(const psVector *exptime, ///< Exposure times for each exposure + const psVector *counts, ///< Counts for each exposure + const psVector *cntError, ///< Error in the counts + const psVector *mask, ///< Mask for each exposure + float offref, ///< Reference time offset + int nIter, ///< Number of iterations + float rej, ///< Rejection threshold (sigma) + psMaskType maskVal ///< Mask value + ); + +/// Generate shutter correction based on a full non-linear fit +/// +/// Performs a full non-linear fit to counts as a function of exposure time. The main purpose is to solve for +/// the reference time offset, so that future fits may be performed using linear fitting with the reference +/// time offset fixed. +pmShutterCorrection *pmShutterCorrectionFullFit(const psVector *exptime, ///< Exposure times for each exposure + const psVector *counts, ///< Counts for each exposure + const psVector *cntError, ///< Error in the counts + const pmShutterCorrection *guess ///< Initial guess + ); + +/// Measure a shutter correction image from an array of images +/// +/// Given an array of readouts (with known exposure times from the cell concepts), this function measures the +/// shutter correction (our principal concern is for the time offset, rather than the normalisation) by +/// measuring the reference time offset using the full non-linear fit for a small number of representative +/// regions (middle and corners), and then using that to perform a linear fit to each pixel. +bool pmShutterCorrectionMeasure(pmReadout *output, ///< Output readout + const psArray *readouts, ///< Array of readouts + int size, ///< Size of samples for statistics for non-linear fit + psStatsOptions meanStat, ///< Statistic to use for mean + psStatsOptions stdevStat, ///< Statistic to use for stdev + int nIter, ///< Number of iterations + float rej, ///< Rejection threshold (sigma) + psMaskType maskVal ///< Mask value + ); + +/// Apply a shutter correction +/// +/// Given a shutter correction (with dT for each pixel), applies this correction to an input image. +bool pmShutterCorrectionApply(pmReadout *readout, ///< Readout to which to apply shutter correction + const pmReadout *shutter ///< Shutter correction readout, with dT for each pixel + ); + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Functions for doing the shutter correction piece-meal (don't have to read entire image stack into memory at +// once). A single read run through the stack is required, calling pmShutterCorrectionAddReadout on each. +// Then pmShutterCorrectionReference provides the required reference shutter time, so that +// pmShutterCorrectionGenerate can generate a shutter correction piece by piece as overlapping pixels from +// each input are read in. + + +/// Data for measuring the shutter correction +typedef struct { + int num; ///< Number of images + int numCols, numRows; ///< Size of images + psArray *regions; ///< Regions at which to measure statistics + psArray *mean; ///< Vector of means at each region + psArray *stdev; ///< Vector of standard deviations at each region + psVector *exptimes; ///< Exposure times for each image + psVector *refs; ///< Reference fluxes +} pmShutterCorrectionData; + +/// Allocator for pmShutterCorrectionData +pmShutterCorrectionData *pmShutterCorrectionDataAlloc(int numCols, int numRows, ///< Size of images + int size ///< Size of regions + ); + +/// Add a readout to the correction data +/// +/// Performs statistics on the readout, recording the data +bool pmShutterCorrectionAddReadout(pmShutterCorrectionData *data, ///< Correction data + const pmReadout *readout, ///< Readout to add + psStatsOptions meanStat, ///< Statistic to use for mean + psStatsOptions stdevStat, ///< Statistic to use for stdev + psMaskType maskVal, ///< Mask value + psRandom *rng ///< Random number generator + ); + +/// Calculate the reference shutter time from the correction data +float pmShutterCorrectionReference(const pmShutterCorrectionData *data ///< Correction data + ); + +/// Generate a shutter correction +/// +/// Performs the linear fit to each pixel in the stack. +bool pmShutterCorrectionGenerate(pmReadout *shutter, ///< Shutter correction + pmReadout *pattern, ///< Background pattern (or NULL) + const psArray *inputs, ///< Stack of input pmReadouts + float reference, ///< Reference shutter time (from pmShutterCorrectionRef) + const pmShutterCorrectionData *data, ///< Correction data + int nIter, ///< Number of iterations + float rej, ///< Rejection threshold (sigma) + psMaskType maskVal ///< Mask value + ); + + + +/// @} +#endif Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmSkySubtract.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmSkySubtract.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmSkySubtract.c (revision 22293) @@ -0,0 +1,738 @@ +/** @file pmSubtractSky.c + * + * This file will contain a module which will create a model of the + * background sky and subtract that from the input image. + * + * @author GLG, MHPCC + * + * @version $Revision: 1.3 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-04-04 22:42:48 $ + * + * Copyright 2004 Maui High Performance Computing Center, University of Hawaii + * + * + * + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmSubtractSky.h" + +// XXX: Get rid of the. Create pmUtils.h +psImage *p_psDetermineTrimmedImage( + pmReadout *in +); + +/****************************************************************************** +DetermineNumBits(data): This routine takes an enum psStatsOptions as an +argument and returns the number of non-zero bits. + +XXX: This code is duplicated in the ReadoutCombine file. + *****************************************************************************/ +static psS32 DetermineNumBits(psStatsOptions data) +{ + psTrace("psModules.detrend", 4, "Calling DetermineNumBits(0x%x)\n", data); + + psS32 i; + psU64 tmpData = data; + psS32 numBits = 0; + + for (i=0;i<(8 * sizeof(psStatsOptions));i++) { + if (0x0001 & tmpData) { + numBits++; + } + tmpData = tmpData >> 1; + } + + psTrace("psModules.detrend", 4, + "Calling DetermineNumBits(0x%x) -> %d\n", data, numBits); + return(numBits); +} + +/****************************************************************************** +getHighestPriorityStatOption(statOptions): this routine takes as input a +psStats->options with multiple options set and returns one with a single +option set according to the precedence set in the SDRS. + *****************************************************************************/ +static psU64 getHighestPriorityStatOption(psU64 statOptions) +{ + psTrace("psModules.detrend", 4, + "Calling getHighestPriorityStatOption(0x%x)\n", statOptions); + + if (statOptions & PS_STAT_SAMPLE_MEAN) { + return(PS_STAT_SAMPLE_MEAN); + } else if (statOptions & PS_STAT_SAMPLE_MEDIAN) { + return(PS_STAT_SAMPLE_MEDIAN); + } else if (statOptions & PS_STAT_CLIPPED_MEAN) { + return(PS_STAT_CLIPPED_MEAN); + } else if (statOptions & PS_STAT_FITTED_MEAN) { + return(PS_STAT_FITTED_MEAN); + } else if (statOptions & PS_STAT_ROBUST_MEDIAN) { + return(PS_STAT_ROBUST_MEDIAN); + } + psError(PS_ERR_UNKNOWN, true, "Unallowable option requested for statistically binning image pixels.\n"); + return(-1); + // XXX + //else if (statOptions & PS_STAT_ROBUST_MODE) { + // return(PS_STAT_ROBUST_MODE); + //} +} + +/****************************************************************************** +psImage *binImage(origImage, binFactor, statOptions): This routine takes an +input psImage and scales it smaller by a factor of binFactor. The statistic +used in combining input pixels is specified in statOptions. + +XXX: use static vectors for myStats, binVector and binMask. +XXX: I coded this before I was aware of a psLib reBin function. I don't +use this function in this module. I'm keeping it here in the event that +requirements change and we might need a custom reBin function. + *****************************************************************************/ +#if 0 +static psImage *binImage(psImage *origImage, + int binFactor, + psStatsOptions statOptions) +{ + psTrace("psModules.detrend", 4, "Calling binImage(%d)\n", binFactor); + + if (binFactor <= 0) { + psLogMsg(__func__, PS_LOG_WARN, + "WARNING: binImage(): binFactor is %d\n", binFactor); + return(origImage); + } + if (binFactor == 1) { + return(origImage); + } + + psVector *binVector = psVectorAlloc(binFactor * binFactor, PS_TYPE_F32); + psVector *binMask = psVectorAlloc(binFactor * binFactor, PS_TYPE_U8); + psStats *myStats = psStatsAlloc(statOptions); + + for (psS32 row = 0; row < origImage->numRows ; row+=binFactor) { + for (psS32 col = 0; col < origImage->numCols ; col+=binFactor) { + psS32 count = 0; + for (psS32 binRow = 0; binRow <= binFactor ; binRow++) { + for (psS32 binCol = 0; binCol <= binFactor ; binCol++) { + if (((row + binRow) < origImage->numRows) && + ((col + binCol) < origImage->numCols)) { + binVector->data.F32[count] = + origImage->data.F32[row + binRow][col + binCol]; + binMask->data.U8[count] = 0; + } else { + binVector->data.F32[count] = 0.0; + binMask->data.U8[count] = 1; + } + count++; + } + } + psStats *rc1 = psVectorStats(myStats, binVector, NULL, binMask, 1); + if (rc1 == NULL) { + psError(PS_ERR_UNKNOWN, false, "psVectorStats(): could not perform requested statistical operation. Returning in image.\n"); + return(origImage); + } + psF64 statValue; + psBool rc = p_psGetStatValue(rc1, &statValue); + + if (rc == true) { + origImage->data.F32[row][col] = (psF32) statValue; + } else { + origImage->data.F32[row][col] = 0.0; + psLogMsg(__func__, PS_LOG_WARN, + "WARNING: pmSubtractSky(), binImage(): p_psGetStatValue() was FALSE\n"); + } + } + } + psFree(binVector); + psFree(binMask); + psFree(myStats); + + psTrace("psModules.detrend", 4, "Exiting binImage(%d)\n", binFactor); + return(origImage); +} +#endif + +/****************************************************************************** +CalculatePolyTerms(xOrder, yOrder): this routine will calculate the number of +coefficients (or terms) in a 2-D polynomial of order (xOrder, yOrder). + +XXX: Use your brain and figure out the analytical expression. + +XXX: Why isn't it simply (xOrder+1) * (yOrder+1)? + *****************************************************************************/ +static psS32 CalculatePolyTerms(psS32 xOrder, psS32 yOrder) +{ + psTrace("psModules.detrend", 4, + "Calling CalculatePolyTerms(%d, %d)\n", xOrder, yOrder); + + psS32 maxOrder = PS_MAX(xOrder, yOrder); + psS32 localPolyTerms = 0; + psS32 order = 0; + psS32 num=0; + + for (order=0;order<=maxOrder;order++) { + for (num=0;num<=order;num++) { + if (((order-num) <= xOrder) && (num <= yOrder)) { + localPolyTerms++; + } + } + } + psTrace("psModules.detrend", 4, + "Exiting CalculatePolyTerms(%d, %d) -> %d\n", xOrder, yOrder, localPolyTerms); + return(localPolyTerms); + + // return((xOrder+1) * (yOrder+1)); +} + +/****************************************************************************** +buildPolyTerms(): this routine computes a 2-D array polyTerms[][] that holds +terms for the polynomial that is used to model the sky background. We use +this array primarily for convenience in computations involving sky model +polynomials. It is defined as: + polyTerms[i][0] = the power to which X is raised in the i-th term of in an + poly-order sky background polynomial. + + polyTerms[i][1] = the power to which Y is raised in the i-th term of in an + poly-order sky background polynomial. + *****************************************************************************/ +static psS32 **buildPolyTerms(psS32 xOrder, psS32 yOrder) +{ + psTrace("psModules.detrend", 4, + "Calling buildPolyTerms(%d, %d)\n", xOrder, yOrder); + + psS32 i=0; + psS32 order = 0; + psS32 num=0; + psS32 localPolyTerms = CalculatePolyTerms(xOrder, yOrder); + psS32 maxOrder = PS_MAX(xOrder, yOrder); + + // Create the data structure which we hold the xy order of each coeff. + psS32 **polyTerms = (psS32 **) psAlloc(localPolyTerms * sizeof(psS32 *)); + for (i=0; i < localPolyTerms ; i++) { + polyTerms[i] = (psS32 *) psAlloc(2 * sizeof(psS32)); + } + + i=0; + // This code segment loops through each term i in the polynomial and + // calculates the power to which x/y are raised in that i-th term. + // We first do the 0-order terms, then the 1-order terms, etc. + for (order=0;order<=maxOrder;order++) { + for (num=0;num<=order;num++) { + if (((order-num) <= xOrder) && (num <= yOrder)) { + polyTerms[i][0] = order-num; + polyTerms[i][1] = num; + i++; + } + } + } + + if (psTraceGetLevel("psModules.detrend") >= 10) { + for (i=0; i < localPolyTerms ; i++) { + printf("x^%d * y^%d\n", polyTerms[i][0], polyTerms[i][1]); + } + } + + psTrace("psModules.detrend", 4, + "Exiting buildPolyTerms(%d, %d)\n", xOrder, yOrder); + return(polyTerms); +} + +/****************************************************************************** +This procedure calculates various combinations of powers of x and y and stores +them in the data structure p_psPolySums[][]. After it completes: + + p_psPolySums[i][j] == x^i * y^j + +XXX: Use a psImage for the p_psPolySums data structure? +XXX: p_psPolySums: should this be a global? Did you get the storage classifier + and name correct? +XXX: Use variable size arrays for p_psPolySums[][]. +XXX: Must initialize p_psPolySums[][]? + *****************************************************************************/ +#define PS_MAX_POLYNOMIAL_ORDER 20 + +psF64 p_psPolySums[PS_MAX_POLYNOMIAL_ORDER+1][PS_MAX_POLYNOMIAL_ORDER+1]; +static void buildSums(psF64 x, + psF64 y, + psS32 xOrder, + psS32 yOrder) +{ + psTrace("psModules.detrend", 4, + "Calling buildPolyTerms(%d, %d)\n", xOrder, yOrder); + + psS32 i = 0; + psS32 j = 0; + psF64 xSum = 0.0; + psF64 ySum = 0.0; + + xSum = 1.0; + ySum = 1.0; + for(i=0;i<=xOrder;i++) { + ySum = xSum; + for(j=0;j<=yOrder;j++) { + p_psPolySums[i][j] = ySum; + ySum*= y; + } + xSum*= x; + } + psTrace("psModules.detrend", 4, + "Exiting buildPolyTerms(%d, %d)\n", xOrder, yOrder); +} + +/****************************************************************************** +ImageFitPolynomial(myPoly, dataImage, maskImage): this private routine takes +an input image along with a mask and fits a polynomial to it. The degree of +the polynomial is specified by input parameter myPoly, and need not be +symmetrical in orders of X and Y. The polynomial must be type +PS_POLYNOMIAL_ORD. If there are not enough rows or columns in the input image +for the order of the polynomial, then that order is reduced. The algorithm +used in this routine is based on that of the pilot project ADD, but is not +documented anywhere. + +XXX: Different trace message facilities in use here. + *****************************************************************************/ +static psPolynomial2D *ImageFitPolynomial( + psPolynomial2D *myPoly, + psImage *dataImage, + psImage *maskImage) +{ + psTrace("psModules.detrend", 4, + "Calling ImageFitPolynomial()\n"); + PS_ASSERT_POLY_NON_NULL(myPoly, NULL); + PS_ASSERT_POLY_TYPE(myPoly, PS_POLYNOMIAL_ORD, NULL); + PS_ASSERT_IMAGE_NON_NULL(dataImage, NULL); + PS_ASSERT_IMAGE_NON_EMPTY(dataImage, NULL); + PS_ASSERT_IMAGE_TYPE(dataImage, PS_TYPE_F32, NULL); + PS_ASSERT_IMAGE_NON_NULL(maskImage, NULL); + PS_ASSERT_IMAGE_NON_EMPTY(maskImage, NULL); + PS_ASSERT_IMAGE_TYPE(maskImage, PS_TYPE_U8, NULL); + PS_ASSERT_IMAGES_SIZE_EQUAL(dataImage, maskImage, NULL); + psS32 oldPolyX = -1; + psS32 oldPolyY = -1; + + // The matrix equations become singular if there are more powers of X + // in myPoly then there are rows of the image. I think. Similarly for + // powers of Y and columns. So. Here we reduce the complexity of the + // polynomial if there are not enough rows/columns in the input image. + + if ((myPoly->nX + 1) > dataImage->numRows) { + psLogMsg(__func__, PS_LOG_WARN, + "WARNING: ImageFitPolynomial(): Reducing polynomial complexity in x-dimension.\n"); + oldPolyX = myPoly->nX; + myPoly->nX = dataImage->numRows - 1; + } + if ((myPoly->nY + 1) > dataImage->numCols) { + psLogMsg(__func__, PS_LOG_WARN, + "WARNING: ImageFitPolynomial(): Reducing polynomial complexity in y-dimension.\n"); + oldPolyY = myPoly->nY; + myPoly->nY = dataImage->numCols - 1; + } + psS32 i; + psS32 j; + psS32 x; + psS32 y; + psS32 aRow; + psS32 aCol; + psS32 **polyTerms = buildPolyTerms(myPoly->nX, myPoly->nY); + // We determine how many coefficients will be in the polynomial that we + // are fitting to this image. + psS32 localPolyTerms = CalculatePolyTerms(myPoly->nX, myPoly->nY); + psImage *A = psImageAlloc(localPolyTerms, localPolyTerms, PS_TYPE_F64); + psImage *Aout = psImageAlloc(localPolyTerms, localPolyTerms, PS_TYPE_F64); + psVector *B = psVectorAlloc(localPolyTerms, PS_TYPE_F64); + psVector *outPerm = NULL; + + // + // Initialize A matrix and B vector. + // + psImageInit(A, 0.0); + psVectorInit(B, 0.0); + + // + // We build the A matrix and B vector. + // + for (x=0;xnumRows;x++) { + for (y=0;ynumCols;y++) { + if (maskImage->data.U8[x][y] == 0) { + buildSums((psF64) x, (psF64) y, myPoly->nX, myPoly->nY); + + /************************************************************ + This code dervies from equation (7) of the pilot ADD. However, + it is not exactly the same in that the order of the polynomial + may be different in X And Y. + + Equation (7) from the pilot ADD describes 16 linear equations. + The i-th equation is simply the partial derivative of the + sky background polynomial (1) w.r.t. to the i-th term in + that polynomial. The i-th equation is stored in row i of + matrix A[][] (matrix A[][] has origin (1,1), not (0,0)). To + compute A[i][j] we simply multiply the j-th term of the Sky + Background Polynomial (SBP) by the i-th term of SBP. + ************************************************************/ + for (aRow=0;aRowdata.F64[aRow][aCol]+= + (p_psPolySums[ polyTerms[aCol][0] ][ polyTerms[aCol][1] ] * + p_psPolySums[ polyTerms[aRow][0] ][ polyTerms[aRow][1] ]); + } + } + // Build the B[] vector, which is the right-hand side of (7). + for (i=0;idata.F64[i]+= dataImage->data.F32[x][y] * + p_psPolySums[ polyTerms[i][0] ][ polyTerms[i][1] ]; + } + } + } + } + + if (psTraceGetLevel(".psModule.pmSubtractSky.ImageFitPolynomial") >= 8) { + for (aRow=0;aRowdata.F64[aRow][aCol]); + } + } + + for (i=0;i<=localPolyTerms;i++) { + printf("B[%d] is %f\n", i, B->data.F64[i]); + } + } + + // + // Solve the matrix equations for the polynomial coefficients C. + // XXX: How do we know if these matrix operations were successful? + // + Aout = psMatrixLUD(Aout, &outPerm, A); + PS_ASSERT_IMAGE_NON_NULL(Aout, NULL); + PS_ASSERT_IMAGE_NON_EMPTY(Aout, NULL); + psVector *C = psVectorAlloc(localPolyTerms, PS_TYPE_F64); + psMatrixLUSolve(C, Aout, B, outPerm); + + // + // Set the appropriate coefficients in the myPoly structure. + // + for (i=0;icoeff[ polyTerms[i][0] ][ polyTerms[i][1] ] = C->data.F64[i]; + psTrace("psModules.detrend", 6, + "myPoly->coeff[%d][%d] is %f\n", polyTerms[i][0], polyTerms[i][1], myPoly->coeff[ polyTerms[i][0] ][ polyTerms[i][1] ]); + } + + // + // Free data structures that were allocated in this module. + // + for (i=0;inX = oldPolyX; + for (i=oldPolyX ; i < (1 + myPoly->nX) ; i++) { + for (j=0;j<(1 + myPoly->nY) ; j++) { + myPoly->coeff[i][j] = 0.0; + } + } + } + if (oldPolyY != -1) { + myPoly->nY = oldPolyY; + for (i=0 ; i < (1 + myPoly->nX) ; i++) { + for (j=oldPolyY;j < (1 + myPoly->nY) ; j++) { + myPoly->coeff[i][j] = 0.0; + } + } + } + + psTrace("psModules.detrend", 4, + "Exiting ImageFitPolynomial()\n"); + // psTrace("psModules.detrend", 4, + // "---- ImageFitPolynomial() end successfully ----\n"); + return(myPoly); +} + + +/****************************************************************************** +pmReadout pmSubtractSky(): + +XXX: use static vectors for myStats, and the binned image + +XXX: The SDR is silent about types. PS_TYPE_F32 is implemented here. + +XXX: Sync the psTrace message facilities. + *****************************************************************************/ +pmReadout *pmSubtractSky(pmReadout *in, + void *fitSpec, + psFit fit, + psS32 binFactor, + psStats *stats, + psF32 clipSD) +{ + PS_ASSERT_READOUT_NON_NULL(in, NULL); + PS_ASSERT_READOUT_NON_EMPTY(in, NULL); + PS_ASSERT_READOUT_TYPE(in, PS_TYPE_F32, NULL); + PS_WARN_PTR_NON_NULL(in->parent); + if (in->parent != NULL) { + PS_WARN_PTR_NON_NULL(in->parent->concepts); + } + psTrace("psModules.detrend", 4, + "---- pmSubtractSky() begin ----\n"); + + if ((fit != PM_FIT_NONE) && + (fit != PM_FIT_POLYNOMIAL) && + (fit != PM_FIT_SPLINE)) { + psError(PS_ERR_UNKNOWN, true, "psFit is unallowable (%d). Returning in image.\n", fit); + return(in); + } + + psStatsOptions statOptions = 0; + + // + // Return the original input readout if the fit specs are poorly defined. + // No warning or error messages should be generated. + // + if ((fitSpec == NULL) || + ((fit == PM_FIT_NONE) || (fit == PM_FIT_SPLINE))) { + // psLogMsg(__func__, PS_LOG_WARN, "Fit specs are poorly defined. Returning in image.\n"); + return(in); + } + + // + // Determine trimmed image from metadata. + // + + psImage *trimmedImg = p_psDetermineTrimmedImage(in); + psImage *binnedImage = NULL; + psPolynomial2D *myPoly = NULL; + psImage *binnedMaskImage = NULL; + psU32 oldStatOptions = 0; + + // + // Determine which statistic to use when binning pixels, if any. + // + if (stats != NULL) { + statOptions = stats->options; + if (1 < DetermineNumBits(statOptions)) { + psLogMsg(__func__, PS_LOG_WARN, "WARNING: Multiple statistical options have been requested.\n"); + statOptions = getHighestPriorityStatOption(statOptions); + if (statOptions == -1) { + psError(PS_ERR_UNKNOWN, true, "Not allowable stats->option was specified. Returning in image.\n"); + return(in); + } + // Save old input "stats" parameter. + oldStatOptions = stats->options; + stats->options = statOptions; + } + if (0 == DetermineNumBits(statOptions)) { + psLogMsg(__func__, PS_LOG_WARN, + "WARNING: pmSubtractSky(): no stats->options was requested\n"); + } + } + + // + // Generate required warning messages. + // + if (binFactor <= 0) { + psLogMsg(__func__, PS_LOG_WARN, + "WARNING: pmSubtractSky(): binFactor is %d\n", binFactor); + } + if (stats == NULL) { + psLogMsg(__func__, PS_LOG_WARN, + "WARNING: pmSubtractSky(): input parameter stats is NULL\n"); + } + + // + // Bin the input image according to input parameters. + // Create a new binned image mask. + // + if ((binFactor <= 1) || (stats == NULL) || (0 == DetermineNumBits(statOptions))) { + // No binning is required here. Simply create a copy of the image + // and a mask. + binnedImage = psImageCopy(binnedImage, trimmedImg, PS_TYPE_F32); + if (binnedImage == NULL) { + psError(PS_ERR_UNKNOWN, false, "psImageCopy() returned NULL. Returning in image.\n"); + return(in); + } + + if (in->mask != NULL) { + binnedMaskImage = psImageCopy(binnedMaskImage, in->mask, PS_TYPE_U8); + if (binnedMaskImage == NULL) { + psError(PS_ERR_UNKNOWN, false, "psImageCopy() returned NULL. Returning in image.\n"); + psFree(binnedImage); + return(in); + } + } else { + binnedMaskImage = psImageAlloc(binnedImage->numCols, + binnedImage->numRows, + PS_TYPE_U8); + psImageInit(binnedMaskImage, 0); + } + } else { + binnedImage = psImageRebin(NULL, trimmedImg, in->mask, 0, binFactor, stats); + if (binnedImage == NULL) { + psError(PS_ERR_UNKNOWN, false, "psImageRebin() returned NULL. Returning in image.\n"); + return(in); + } + binnedMaskImage = psImageAlloc(binnedImage->numCols, + binnedImage->numRows, + PS_TYPE_U8); + psImageInit(binnedMaskImage, 0); + } + psTrace("psModules.detrend", 4, + "binnedImage size is (%d, %d)\n", binnedImage->numRows, binnedImage->numCols); + + // + // Clip pixels that are outside the acceptable range. + // + if (clipSD <= 0.0) { + psLogMsg(__func__, PS_LOG_WARN, + "WARNING: pmSubtractSky(): clipSD is %f\n", clipSD); + } else { + // Determine the mean and standard deviation of the binned image. + psStats *myStats = psStatsAlloc(PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); + if (!psImageStats(myStats, binnedImage, NULL, 0)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Couldn't get statistics for image.\n"); + return NULL; + } + psF64 binnedMean = myStats->sampleMean; + psF64 binnedStdev = myStats->sampleStdev; + psFree(myStats); + psTrace("psModules.detrend", 6, + "binned StDev is %f\n", binnedStdev); + + // Clip all pixels which are more than clipSD sigmas from the mean. + psTrace("psModules.detrend", 6, + "clipSD is %f\n", clipSD); + + for (psS32 row = 0; row < binnedImage->numRows ; row++) { + for (psS32 col = 0; col < binnedImage->numCols ; col++) { + if (fabs(binnedImage->data.F32[row][col] - binnedMean) > + (clipSD * binnedStdev)) { + binnedMaskImage->data.U8[row][col] = 1; + } + } + } + } + + // + // Fit the polynomial to the binned image + // + if (fit == PM_FIT_POLYNOMIAL) { + myPoly = (psPolynomial2D *) fitSpec; + PS_ASSERT_POLY_NON_NULL(myPoly, NULL); + PS_ASSERT_POLY_TYPE(myPoly, PS_POLYNOMIAL_ORD, NULL); + + myPoly = ImageFitPolynomial(myPoly, binnedImage, binnedMaskImage); + + if (myPoly != NULL) { + // Set the pixels in the binned image to that of the polynomial. + binnedImage = psImageEvalPolynomial(binnedImage, myPoly); + if (binnedImage == NULL) { + psError(PS_ERR_UNKNOWN, false, "psImageEvalPolynomial() returned NULL. Returning in image.\n"); + psFree(binnedMaskImage); + if (!((binFactor <= 1) || (stats == NULL))) { + psFree(binnedImage); + } + if (oldStatOptions != 0) { + stats->options = statOptions; + } + return(in); + } + } else { + psLogMsg(__func__, PS_LOG_WARN, + "WARNING: pmSubtractSky(): could not model sky with a polynomial.\n"); + psFree(binnedMaskImage); + if (!((binFactor <= 1) || (stats == NULL))) { + psFree(binnedImage); + } + if (oldStatOptions != 0) { + stats->options = statOptions; + } + return(in); + } + } else { + // We shouldn't get here since we check this above. + psError(PS_ERR_UNKNOWN, true, "Unallowable fit type. Returning in image.\n"); + psFree(binnedMaskImage); + if (!((binFactor <= 1) || (stats == NULL))) { + psFree(binnedImage); + } + if (oldStatOptions != 0) { + stats->options = statOptions; + } + return(in); + } + + // + //Subtract the polynomially fitted image from the original image + // + if (binFactor <= 1) { + // The binned image is the same size as the original image. + for (psS32 row = 0; row < trimmedImg->numRows ; row++) { + for (psS32 col = 0; col < trimmedImg->numCols ; col++) { + trimmedImg->data.F32[row][col]-= binnedImage->data.F32[row][col]; + } + } + } else { + + psImageInterpolateOptions *interp = psImageInterpolateOptionsAlloc(PS_INTERPOLATE_BILINEAR, + binnedImage, NULL, NULL, 0, + 0.0, 0.0, 0, 0, 0.0); + + for (psS32 row = 0; row < trimmedImg->numRows ; row++) { + for (psS32 col = 0; col < trimmedImg->numCols ; col++) { + // We calculate the F32 value of the pixel coordinates in the + // binned image and then use a pixel interpolation routine to + // determine the value of the pixel at that location. + psF32 binRowF64 = ((psF32) row) / ((psF32) binFactor); + psF32 binColF64 = ((psF32) col) / ((psF32) binFactor); + + // We add 0.5 to the pixel locations since the pixel + // interpolation routine defines the location of pixel + // (i, j) as (i+0.5, j+0.5). + binRowF64+= 0.5; + binColF64+= 0.5; + + double binPixel; + if (!psImagePixelInterpolate(&binPixel, NULL, NULL, binColF64, binRowF64, interp)) { + psError(PS_ERR_UNKNOWN, false, "Unable to interpolate image."); + psFree(interp); + psFree(binnedImage); + return NULL; + } + trimmedImg->data.F32[row][col] -= binPixel; + + psTrace("psModules.detrend", 8, + "image[%d][%d] <--> binnedImage[%.2f][%.2f]: %lf\n", + row, col, binRowF64-0.5, binColF64-0.5, binPixel); + } + } + psFree(interp); + + } + psFree(binnedMaskImage); + psFree(binnedImage); + if (oldStatOptions != 0) { + stats->options = statOptions; + } + + psTrace("psModules.detrend", 4, + "---- pmSubtractSky() exit successfully ----\n"); + return(in); +} Index: /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmSkySubtract.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmSkySubtract.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/detrend/pmSkySubtract.h (revision 22293) @@ -0,0 +1,34 @@ +/* @file pmSubtractSky.h + * + * This file will contain a module which will create a model of the + * background sky and subtract that from the input image. + * + * @author GLG, MHPCC + * + * @version $Revision: 1.3 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-03-30 21:12:56 $ + * Copyright 2004 Maui High Performance Computing Center, University of Hawaii + */ + +#ifndef PM_SUBTRACT_SKY_H +#define PM_SUBTRACT_SKY_H + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +// XXX: this is pmFit in pmSubtractBias.c, named psFit here. +typedef enum { + PM_FIT_NONE, ///< No fit + PM_FIT_POLYNOMIAL, ///< Fit polynomial + PM_FIT_SPLINE ///< Fit cubic splines +} psFit; + +pmReadout *pmSubtractSky(pmReadout *in, + void *fitSpec, + psFit fit, + int binFactor, + psStats *stats, + float clipSD); + +/// @} +#endif Index: /tags/pap_tags/pap_root_080320/psModules/src/imcombine/pmReadoutCombine.c =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/imcombine/pmReadoutCombine.c (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/imcombine/pmReadoutCombine.c (revision 22293) @@ -0,0 +1,356 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmHDUUtils.h" +#include "pmFPAMaskWeight.h" +#include "pmConceptsAverage.h" +#include "pmReadoutStack.h" + +#include "pmReadoutCombine.h" + +//#define SHOW_BUSY 1 // Show that the function is busy + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Public functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Allocator for pmCombineParams +pmCombineParams *pmCombineParamsAlloc(psStatsOptions combine) +{ + pmCombineParams *params = psAlloc(sizeof(pmCombineParams)); + + params->combine = combine; + params->maskVal = 0; + params->blank = 0; + params->nKeep = 0; + params->fracHigh = 0.0; + params->fracHigh = 0.0; + params->iter = 1; + params->rej = INFINITY; + params->weights = false; + + return params; +} + + +// XXX: Maybe add support for S16 and S32 types. Currently, only F32 supported. +bool pmReadoutCombine(pmReadout *output, const psArray *inputs, const psVector *zero, const psVector *scale, + const pmCombineParams *params) +{ + // Check inputs + PS_ASSERT_PTR_NON_NULL(output, false); + PS_ASSERT_ARRAY_NON_NULL(inputs, false); + PS_ASSERT_PTR_NON_NULL(params, false); + if (zero) { + PS_ASSERT_VECTOR_TYPE(zero, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_SIZE(zero, inputs->n, false); + } + if (scale) { + PS_ASSERT_VECTOR_TYPE(scale, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_SIZE(scale, inputs->n, false); + } + PS_ASSERT_FLOAT_WITHIN_RANGE(params->fracLow, 0.0, 1.0, false); + PS_ASSERT_FLOAT_WITHIN_RANGE(params->fracHigh, 0.0, 1.0, false); + if (params->combine != PS_STAT_SAMPLE_MEAN && params->combine != PS_STAT_SAMPLE_MEDIAN && + params->combine != PS_STAT_ROBUST_MEDIAN && params->combine != PS_STAT_FITTED_MEAN && + params->combine != PS_STAT_CLIPPED_MEAN) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Combination method is not SAMPLE_MEAN, SAMPLE_MEDIAN, " + "ROBUST_MEDIAN, FITTED_MEAN or CLIPPED_MEAN.\n"); + return false; + } + for (int i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + if (params->weights && !readout->weight) { + psError(PS_ERR_UNEXPECTED_NULL, true, + "Rejection based on weights requested, but no weights supplied for image %d.\n", i); + return false; + } + } + + bool first = !output->image; // First pass through? + + pmHDU *hdu = pmHDUFromReadout(output); // Output HDU + if (!hdu) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find HDU for readout.\n"); + return false; + } + + if (first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Combining using statistic: %x", params->combine); + if (!hdu->header) { + hdu->header = psMetadataAlloc(); + } + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + psStats *stats = psStatsAlloc(params->combine); // The statistics to use in the combination + if (params->combine == PS_STAT_CLIPPED_MEAN) { + stats->clipSigma = params->rej; + stats->clipIter = params->iter; + + if (first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Combination clipping: %d iterations, rejection at %f sigma", + params->iter, params->rej); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + } + + int minInputCols, maxInputCols, minInputRows, maxInputRows; // Smallest and largest values to combine + int xSize, ySize; // Size of the output image + if (!pmReadoutStackValidate(&minInputCols, &maxInputCols, &minInputRows, &maxInputRows, &xSize, &ySize, + inputs)) { + psError(PS_ERR_UNKNOWN, false, "No valid input readouts."); + return false; + } + + pmReadoutUpdateSize(output, minInputCols, minInputRows, xSize, ySize, true, params->weights, + params->blank); + psTrace("psModules.imcombine", 7, "Output minimum: %d,%d\n", output->col0, output->row0); + + psStatsOptions combineStdev = 0; // Statistics option for weights + if (params->weights) { + if (first) { + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + "Using input weights to combine images", ""); + } + + // Get the correct statistics option for weights + switch (params->combine) { + case PS_STAT_SAMPLE_MEAN: + case PS_STAT_SAMPLE_MEDIAN: + combineStdev = PS_STAT_SAMPLE_STDEV; + break; + case PS_STAT_ROBUST_MEDIAN: + combineStdev = PS_STAT_ROBUST_STDEV; + break; + case PS_STAT_FITTED_MEAN: + combineStdev = PS_STAT_FITTED_STDEV; + break; + case PS_STAT_CLIPPED_MEAN: + combineStdev = PS_STAT_CLIPPED_STDEV; + break; + default: + psAbort("Should never get here --- checked params->combine before.\n"); + } + stats->options |= combineStdev; + } + + // We loop through each pixel in the output image. We loop through each input readout. We determine if + // that output pixel is contained in the image from that readout. If so, we save it in psVector pixels. + // If not, we set a mask for that element in pixels. Then, we mask off pixels not between fracLow and + // fracHigh. Then we call the vector stats routine on those pixels/mask. Then we set the output pixel + // value to the result of the stats call. + + psVector *pixels = psVectorAlloc(inputs->n, PS_TYPE_F32); // Stack of pixels + psF32 *pixelsData = pixels->data.F32; // Dereference pixels + + psVector *mask = psVectorAlloc(inputs->n, PS_TYPE_U8); // Mask for stack + psU8 *maskData = mask->data.U8; // Dereference mask + + psVector *weights = NULL; // Stack of weights + psVector *errors = NULL; // Stack of errors (sqrt of variance/weights), for psVectorStats + psF32 *weightsData = NULL; // Dereference weights + if (params->weights) { + weights = psVectorAlloc(inputs->n, PS_TYPE_F32); // Stack of weights + weightsData = weights->data.F32; + } + psVector *index = NULL; // The indices to sort the pixels + + float keepFrac = 1.0 - params->fracLow - params->fracHigh; // Fraction of pixels to keep + if (keepFrac != 1.0 && first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Min/max rejection: %f high, %f low, keep %d", + params->fracHigh, params->fracLow, params->nKeep); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + psMaskType maskVal = params->maskVal; // The mask value + if (maskVal && first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Mask for combination: %x", maskVal); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + #ifndef PS_NO_TRACE + psTrace("psModules.imcombine", 3, "Iterating output: %d --> %d, %d --> %d\n", + minInputCols - output->col0, maxInputCols - output->col0, + minInputRows - output->row0, maxInputRows - output->row0); + if (psTraceGetLevel("psModules.imcombine") >= 3) { + for (int r = 0; r < inputs->n; r++) { + pmReadout *readout = inputs->data[r]; // Input readout + psTrace("psModules.imcombine", 3, "Iterating input %d: %d --> %d, %d --> %d\n", r, + minInputCols - readout->col0, maxInputCols - readout->col0, + minInputRows - readout->row0, maxInputRows - readout->row0); + } + } + #endif + + // Dereference output products + psF32 **outputImage = output->image->data.F32; // Output image + psU8 **outputMask = output->mask->data.U8; // Output mask + psF32 **outputWeight = NULL; // Output weight map + if (output->weight) { + outputWeight = output->weight->data.F32; + } + + psVector *invScale = NULL; // Inverse scale; pre-calculated for efficiency + if (scale) { + invScale = (psVector*)psBinaryOp(NULL, psScalarAlloc(1.0, PS_TYPE_F32), "/", (const psPtr)scale); + } + + for (int i = minInputRows; i < maxInputRows; i++) { + int yOut = i - output->row0; // y position on output readout + #ifdef SHOW_BUSY + + if (psTraceGetLevel("psModules.imcombine") > 9) { + printf("Processing row %d\r", i); + fflush(stdout); + } + #endif + for (int j = minInputCols; j < maxInputCols; j++) { + int xOut = j - output->col0; // x position on output readout + + int numValid = 0; // Number of valid pixels in the stack + memset(maskData, 0, mask->n * sizeof(psU8)); // Reset the mask + for (int r = 0; r < inputs->n; r++) { + pmReadout *readout = inputs->data[r]; // Input readout + int yIn = i - readout->row0; // y position on input readout + int xIn = j - readout->col0; // x position on input readout + psImage *image = readout->image; // The readout image + + #if 0 // This should have been taken care of already: + // Check bounds + if (xIn < 0 || xIn >= image->numCols || yIn < 0 || yIn >= image->numRows) { + continue; + } + #endif + + pixelsData[r] = image->data.F32[yIn][xIn]; + if (!isfinite(pixelsData[r])) { + maskData[r] = 1; + continue; + } + + // Check mask + psImage *roMask = readout->mask; // The mask image + if (roMask && roMask->data.U8[yIn][xIn] & maskVal) { + maskData[r] = 1; + continue; + } + + if (params->weights) { + weightsData[r] = readout->weight->data.F32[yIn][xIn]; + } + + if (zero) { + pixelsData[r] -= zero->data.F32[r]; + } + if (scale) { + pixelsData[r] *= invScale->data.F32[r]; + if (params->weights) { + weightsData[r] *= invScale->data.F32[r] * invScale->data.F32[r]; + } + } + + numValid++; + } + + if (numValid == 0) { + outputMask[yOut][xOut] = params->blank; + outputImage[yOut][xOut] = NAN; + continue; + } + + // Apply fracLow,fracHigh if there are enough pixels + if (numValid * keepFrac >= params->nKeep && keepFrac != 1.0) { + index = psVectorSortIndex(index, pixels); + int numLow = numValid * params->fracLow; // Number of low pixels to clip + int numHigh = numValid * params->fracHigh; // Number of high pixels to clip + // Low pixels + psS32 *indexData = index->data.S32; // Dereference index + for (int k = 0, numMasked = 0; numMasked < numLow && k < index->n; k++) { + // Don't count the ones that are already masked + if (!maskData[indexData[k]]) { + maskData[indexData[k]] = 1; + numMasked++; + } + } + // High pixels + for (int k = pixels->n - 1, numMasked = 0; numMasked < numHigh && k >= 0; k--) { + // Don't count the ones that are already masked + if (! maskData[indexData[k]]) { + maskData[indexData[k]] = 1; + numMasked++; + } + } + } + + // XXXXX this step probably is very expensive : convert errors to variance everywhere? + if (params->weights) { + errors = (psVector*)psUnaryOp(errors, weights, "sqrt"); + } + + // Combination + if (!psVectorStats(stats, pixels, errors, mask, 1)) { + // Can't do much about it, but it's not worth worrying about + psErrorClear(); + outputImage[yOut][xOut] = NAN; + outputMask[yOut][xOut] = params->blank; + if (params->weights) { + outputWeight[yOut][xOut] = NAN; + } + } else { + outputImage[yOut][xOut] = psStatsGetValue(stats, params->combine); + outputMask[yOut][xOut] = isfinite(outputImage[yOut][xOut]) ? 0 : params->blank; + if (params->weights) { + float stdev = psStatsGetValue(stats, combineStdev); + outputWeight[yOut][xOut] = PS_SQR(stdev); // Variance + // XXXX this is not the correct formal error. + // also, the weighted mean is not obviously the correct thing here + } + } + } + } + #ifdef SHOW_BUSY + if (psTraceGetLevel("psModules.imcombine") > 9) { + printf("\n"); + } + #endif + psFree(index); + psFree(pixels); + psFree(mask); + psFree(weights); + psFree(errors); + psFree(stats); + psFree(invScale); + + // Update the "concepts" + psList *inputCells = psListAlloc(NULL); // List of cells + for (long i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + psListAdd(inputCells, PS_LIST_TAIL, readout->parent); + } + bool success = pmConceptsAverageCells(output->parent, inputCells, NULL, NULL, true); + psFree(inputCells); + + output->data_exists = true; + output->parent->data_exists = true; + output->parent->parent->data_exists = true; + + return success; +} + Index: /tags/pap_tags/pap_root_080320/psModules/src/imcombine/pmReadoutCombine.h =================================================================== --- /tags/pap_tags/pap_root_080320/psModules/src/imcombine/pmReadoutCombine.h (revision 22293) +++ /tags/pap_tags/pap_root_080320/psModules/src/imcombine/pmReadoutCombine.h (revision 22293) @@ -0,0 +1,49 @@ +/* @file pmReadoutCombine.h + * @brief Combine multiple readouts + * + * @author George Gusciora, MHPCC + * @author Paul Price, IfA + * + * @version $Revision: 1.12 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-06-02 03:51:03 $ + * Copyright 2004-2006 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_READOUT_COMBINE_H +#define PM_READOUT_COMBINE_H + +/// @addtogroup imcombine Image Combinations +/// @{ + +/// Combination parameters for pmReadoutCombine. +/// +/// These values define how the combination is performed, and should not vary by detector, so that it can be +/// re-used for multiple combinations. +typedef struct +{ + psStatsOptions combine; ///< Statistic to use when performing the combination + psMaskType maskVal; ///< Mask value + psMaskType blank; ///< Mask value to give blank (i.e., no data) pixels + int nKeep; ///< Mimimum number of pixels to keep + float fracHigh; ///< Fraction of high pixels to immediately throw + float fracLow; ///< Fraction of low pixels to immediately throw + int iter; ///< Number of iterations for clipping (for CLIPPED_MEAN only) + float rej; ///< Rejection threshould for clipping (for CLIPPED_MEAN only) + bool weights; ///< Use the supplied weights (instead of calculated stdev)? +} +pmCombineParams; + +// Allocator for pmCombineParams +pmCombineParams *pmCombineParamsAlloc(psStatsOptions statsOptions ///< Statistic to use for combination + ); + +/// Combine multiple readouts, applying zero and scale, with optional minmax clipping +bool pmReadoutCombine(pmReadout *output,///< Output readout; altered and returned + const psArray *inputs, ///< Array of input readouts (F32 image and weight, U8 mask) + const psVector *zero, ///< Zero corrections to subtract from input, or NULL + const psVector *scale, ///< Scale corrections to divide into input, or NULL + const pmCombineParams *params ///< Combination parameters + ); + +/// @} +#endif