Index: /tags/pap_merge_030828/ippScripts/scripts/detrend_stack.pl =================================================================== --- /tags/pap_merge_030828/ippScripts/scripts/detrend_stack.pl (revision 17232) +++ /tags/pap_merge_030828/ippScripts/scripts/detrend_stack.pl (revision 17232) @@ -0,0 +1,255 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $iter, $class_id, $det_type, $camera, $outroot, $dbname, $reduction, $verbose, $save_temps, + $no_update, $no_op ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'camera|c=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class for processing + 'verbose' => \$verbose, # Print to stdout + 'save-temps' => \$save_temps, # Save temporary files? + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --class_id --det_type --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $class_id and + defined $det_type and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_STACK'); # Recipe name to use + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# The output file rule name depends on the detrend type +my $FILERULES = { 'FLATMASK' => 'PPMERGE.OUTPUT.MASK', + 'DARKMASK' => 'PPMERGE.OUTPUT.MASK', + 'MASK' => 'PPMERGE.OUTPUT.MASK', + 'BIAS' => 'PPMERGE.OUTPUT.BIAS', + 'DARK' => 'PPMERGE.OUTPUT.DARK', + 'SHUTTER' => 'PPMERGE.OUTPUT.SHUTTER', + 'FLAT' => 'PPMERGE.OUTPUT.FLAT', + 'DOMEFLAT' => 'PPMERGE.OUTPUT.FLAT', + 'SKYFLAT' => 'PPMERGE.OUTPUT.FLAT', + 'FRINGE' => 'PPMERGE.OUTPUT.FRINGE', + }; +my $output_filerule = $FILERULES->{$det_type}; # File rule for output +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless defined $output_filerule; + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppMerge = can_run('ppMerge') or (warn "Can't find ppMerge" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of files to stack +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -processedimfile -included"; + $command .= " -det_id $det_id"; + $command .= " -class_id $class_id"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $iter, $class_id, $error_code); + } + + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); +} + +# Generate MDC file with the inputs +my ($listFile, $listName) = tempfile( $ipprc->file_resolve("$outroot.$class_id.list.XXXX"), UNLINK => !$save_temps ); +my $num = 0; +foreach my $file (@$files) { + if ($file->{ignored}) { next; } + + print $listFile "INPUT$num\tMETADATA\n"; + $num++; + + my $image = $file->{uri}; # Image name + my $mask = $ipprc->filename( "PPIMAGE.OUTPUT.MASK", $file->{path_base} ); # Mask name + my $weight = $ipprc->filename( "PPIMAGE.OUTPUT.WEIGHT", $file->{path_base} ); # Weight name + + &my_die("Image $image does not exist", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image ); + print $listFile "\tIMAGE\tSTR\t" . $image . "\n"; + + if ($ipprc->file_exists( $mask )) { + print $listFile "\tMASK\tSTR\t" . $mask . "\n"; + } + if ($ipprc->file_exists( $weight )) { + print $listFile "\tWEIGHT\tSTR\t" . $weight . "\n"; + } + + print $listFile "END\n\n"; +} +close $listFile; + + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $outputStack = $ipprc->filename($output_filerule, $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Output name +my $outputCount = $ipprc->filename("PPMERGE.OUTPUT.COUNT", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Count image +my $outputSigma = $ipprc->filename("PPMERGE.OUTPUT.SIGMA", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Stdev image +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Statistics name +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Trace messages +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Log messages + +$command = "$ppMerge $listName $outroot"; # Command to run +$command .= " -recipe PPMERGE $recipe"; +$command .= ' -type ' . uc($det_type); # Type of stacking to perform +$command .= " -stats $outputStats"; # Statistics output filename +$command .= " -recipe PPSTATS CHIPSTATS"; +$command .= " -tracedest $traceDest -log $logDest"; + +# Stack the files +unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppMerge: $error_code", $det_id, $iter, $class_id, $error_code); + } + &my_die("Unable to find expected output file: $outputStack\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStack); + &my_die("Unable to find expected output file: $outputCount\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputCount); + &my_die("Unable to find expected output file: $outputSigma\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputSigma); + &my_die("Unable to find expected output file: $outputStats\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + + # Get the statistics on the stacked image + open(my $statsFile, $ipprc->file_resolve("$outputStats")) or + &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + my $contents = do { local $/; <$statsFile> }; # Contents of file + close($statsFile); + + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse($contents) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + + $stats->parse($metadata) or + &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); +} + +# Command to update the database +$command = "$dettool -addstacked"; +$command .= " -det_id $det_id -iteration $iter"; +$command .= " -class_id $class_id"; +$command .= " -uri $outputStack"; +$command .= " -recip $recipe"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the resultant into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addstacked: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $class_id and not $no_update) { + my $command = "$dettool -addstacked"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /tags/pap_merge_030828/ippconfig/cfh12k/filerules-mef.mdc =================================================================== --- /tags/pap_merge_030828/ippconfig/cfh12k/filerules-mef.mdc (revision 17232) +++ /tags/pap_merge_030828/ippconfig/cfh12k/filerules-mef.mdc (revision 17232) @@ -0,0 +1,191 @@ +### File rules for PHU=FPA, EXT=CHIP + +### Redirections +PPIMAGE.OUTPUT STR PPIMAGE.OUTPUT.MEF +PPIMAGE.OUTPUT.MASK STR PPIMAGE.OUT.MK.MEF +PPIMAGE.OUTPUT.WEIGHT STR PPIMAGE.OUT.WT.MEF + +PPIMAGE.CHIP STR PPIMAGE.CHIP.MEF +PPIMAGE.CHIP.MASK STR PPIMAGE.CHIP.MK.MEF +PPIMAGE.CHIP.WEIGHT STR PPIMAGE.CHIP.WT.MEF + +PPIMAGE.OUTPUT.FPA1 STR PPIMAGE.OUTPUT.FPA1.MEF +PPIMAGE.OUTPUT.FPA2 STR PPIMAGE.OUTPUT.FPA2.MEF +PPIMAGE.BIN1 STR PPIMAGE.BIN1.MEF +PPIMAGE.BIN2 STR PPIMAGE.BIN2.MEF + +PPSTAMP.CHIP STR PPSTAMP.CHIP.MEF + +PSASTRO.INPUT STR PSASTRO.INPUT.CMF +PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.MEF +PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF +PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.MEF + +DVOCORR.OUTPUT STR DVOCORR.MEF.OUTPUT +DVOFLAT.OUTPUT STR DVOFLAT.MEF.OUTPUT + +### input file definitions +### use @DETDB entries to get the detrend images from the database +### replace @DETDB with @FILES if you want to require it from the +### command line, or with an explicit name to require a specific file +TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + +## files used by ppImage +PPIMAGE.INPUT INPUT @FILES CHIP IMAGE +PPIMAGE.MASK INPUT @DETDB CHIP MASK +PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE +PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE +PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE +PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE + +## Files used by ppMerge +PPMERGE.INPUT INPUT @FILES CHIP IMAGE +PPMERGE.INPUT.MASK INPUT @FILES CHIP MASK +PPMERGE.INPUT.WEIGHT INPUT @FILES CHIP WEIGHT + +## files used to build and apply the flat-field correction images +DVOCORR.INPUT INPUT @FILES CHIP IMAGE +DVOCORR.REFHEAD INPUT @FILES CHIP HEADER +DVOFLAT.INPUT INPUT @FILES CHIP IMAGE +DVOFLAT.CORR INPUT @DETDB CHIP IMAGE + +## files used by psphot +PSPHOT.LOAD INPUT @FILES CHIP IMAGE +PSPHOT.INPUT INPUT @FILES CHIP IMAGE +PSPHOT.MASK INPUT @FILES CHIP MASK +PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT +PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF +PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by psastro +PSASTRO.INPUT.CMP INPUT @FILES CHIP CMP +PSASTRO.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by pswarp +PSWARP.INPUT INPUT @FILES CHIP IMAGE +PSWARP.WEIGHT INPUT @FILES CHIP WEIGHT +PSWARP.MASK INPUT @FILES CHIP MASK +PSWARP.SKYCELL INPUT @FILES CHIP IMAGE +PSWARP.ASTROM INPUT @FILES CHIP CMF + +PPSUB.INPUT INPUT @FILES FPA IMAGE +PPSUB.INPUT.MASK INPUT @FILES FPA MASK +PPSUB.INPUT.WEIGHT INPUT @FILES FPA WEIGHT +PPSUB.REF INPUT @FILES FPA IMAGE +PPSUB.REF.MASK INPUT @FILES FPA MASK +PPSUB.REF.WEIGHT INPUT @FILES FPA WEIGHT +PPSUB.SOURCES INPUT @FILES FPA CMF + +PPSTACK.INPUT INPUT @FILES FPA IMAGE +PPSTACK.INPUT.MASK INPUT @FILES FPA MASK +PPSTACK.INPUT.WEIGHT INPUT @FILES FPA WEIGHT +PPSTACK.SOURCES INPUT @FILES FPA CMF + +PPSTAMP.INPUT INPUT @FILES CHIP IMAGE + +PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE +PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + +### output file definitions +TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT +PPIMAGE.OUTPUT.MEF OUTPUT {OUTPUT}.b0.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.OUT.MK.MEF OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK CHIP TRUE MEF +PPIMAGE.OUT.WT.MEF OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT CHIP TRUE MEF +PPIMAGE.OUTPUT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b0.fits IMAGE NONE CHIP TRUE SPLIT +PPIMAGE.OUT.MK.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits MASK NONE CHIP TRUE SPLIT +PPIMAGE.OUT.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT NONE CHIP TRUE SPLIT + +PPIMAGE.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.CHIP.MK.MEF OUTPUT {OUTPUT}.ch.mk.fits MASK COMP_MASK CHIP TRUE MEF +PPIMAGE.CHIP.WT.MEF OUTPUT {OUTPUT}.ch.wt.fits WEIGHT COMP_WT CHIP TRUE MEF +PPIMAGE.CHIP.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE NONE CHIP TRUE SPLIT +PPIMAGE.CHIP.MK.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.mk.fits MASK NONE CHIP TRUE SPLIT +PPIMAGE.CHIP.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.wt.fits WEIGHT NONE CHIP TRUE SPLIT + +PPIMAGE.OUTPUT.FPA1.MEF OUTPUT {OUTPUT}.b1.fits IMAGE COMP_IMG FPA TRUE MEF +PPIMAGE.OUTPUT.FPA2.MEF OUTPUT {OUTPUT}.b2.fits IMAGE COMP_IMG FPA TRUE MEF +PPIMAGE.OUTPUT.FPA1.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE NONE FPA TRUE SPLIT +PPIMAGE.OUTPUT.FPA2.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE NONE FPA TRUE SPLIT + +PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE MEF + +PPIMAGE.BIN1.MEF OUTPUT {OUTPUT}.b1c.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.BIN2.MEF OUTPUT {OUTPUT}.b2c.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.BIN1.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE COMP_IMG CHIP TRUE SPLIT +PPIMAGE.BIN2.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE COMP_IMG CHIP TRUE SPLIT + +PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE + +PPMERGE.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE +PPMERGE.OUTPUT.BIAS OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.DARK OUTPUT {OUTPUT}.fits DARK NONE CHIP TRUE NONE +PPMERGE.OUTPUT.SHUTTER OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.FLAT OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.fits FRINGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.sigma.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.COUNT OUTPUT {OUTPUT}.count.fits IMAGE NONE CHIP TRUE NONE + +DVOCORR.MEF.OUTPUT OUTPUT {OUTPUT}.fc.fits IMAGE NONE CHIP TRUE MEF +DVOCORR.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE NONE CHIP TRUE SPLIT +DVOFLAT.MEF.OUTPUT OUTPUT {OUTPUT}.co.fits IMAGE NONE CHIP TRUE MEF +DVOFLAT.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE NONE CHIP TRUE SPLIT + +PSPHOT.RESID OUTPUT {OUTPUT}.res.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKGND OUTPUT {OUTPUT}.bck.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKSUB OUTPUT {OUTPUT}.sub.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKMDL OUTPUT {OUTPUT}.mdl.fits IMAGE NONE CHIP FALSE MEF + +PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW NONE CHIP TRUE NONE +PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.{CHIP.NAME}.sx SX NONE CHIP TRUE NONE +PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.{CHIP.NAME}.obj OBJ NONE CHIP TRUE NONE +PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.cmp CMP NONE CHIP TRUE NONE +PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF NONE CHIP TRUE SPLIT +PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF NONE CHIP TRUE MEF + +PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE CHIP TRUE MEF + +SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.mnt.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.psf.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.dap.png KAPA NONE FPA TRUE NONE + +PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE +PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE SPLIT +PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUT.MODEL OUTPUT {OUTPUT}.asm ASTROM.MODEL NONE FPA TRUE NONE +PSASTRO.OUT.REFSTARS OUTPUT {OUTPUT}.aref.fits ASTROM.REFSTARS NONE FPA TRUE NONE + +PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT FPA TRUE NONE +PSWARP.OUTPUT.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE +PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE COMP_IMG FPA TRUE NONE +PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE COMP_IMG FPA TRUE NONE + +SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE +SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + +PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT FPA TRUE NONE + +PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_WT FPA TRUE NONE + +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + +PPSIM.OUTPUT.MEF OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE MEF +PPSIM.OUTPUT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT +PPSIM.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE MEF + +PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE COMP_IMG CHIP TRUE NONE +PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK COMP_MASK CHIP TRUE NONE + +LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE +LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE + +TRACE.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.trace TEXT NONE CHIP TRUE NONE +TRACE.EXP OUTPUT {OUTPUT}.trace TEXT NONE FPA TRUE NONE Index: /tags/pap_merge_030828/ippconfig/ctio_mosaic2/camera.config =================================================================== --- /tags/pap_merge_030828/ippconfig/ctio_mosaic2/camera.config (revision 17232) +++ /tags/pap_merge_030828/ippconfig/ctio_mosaic2/camera.config (revision 17232) @@ -0,0 +1,415 @@ +# Camera configuration file for the CTIO MOSAIC2 : describes the camera + +# File formats that we know about +FORMATS METADATA + MEF STR ctio_mosaic2/format.config +END + +# Description of camera --- all the chips and the cells that comprise them +FPA METADATA + ccd00 STR LeftAmp RightAmp + ccd01 STR LeftAmp RightAmp + ccd02 STR LeftAmp RightAmp + ccd03 STR LeftAmp RightAmp + ccd04 STR LeftAmp RightAmp + ccd05 STR LeftAmp RightAmp + ccd06 STR LeftAmp RightAmp + ccd07 STR LeftAmp RightAmp +END + +# menu of possible filter names: internal STR external name +FILTER.ID METADATA + VR MULTI + VR STR VR.Supermacho.c6027 + VR STR VR Supermacho c6027 + VR STR VR SuperMacho c6027 + R STR R Harris c6004 + I STR I c6028 + U STR U c6001 + B STR B Harris c6002 + V STR V Harris c6026 + # Minor filters + g MULTI + g STR g SDSS c6017 + g STR SDSS g c6017 + r MULTI + r STR SDSS r c6018 + r STR r SDSS c6018 + i MULTI + i STR i SDSS c6019 + i STR SDSS i c6019 + # Unused filters (only used in zeros, and therefore irrelevent) + NULL MULTI + NULL STR z SDSS c6020 + NULL STR SDSS z c6020 + NULL STR ha8 H-alpha+8nm c6011 + NULL STR M Washington c6007 + NULL STR u SDSS c6021 + NULL STR VRr Supermacho c6027 +END + +# Table of strings to use for the class identifier (see ippdb) according to the file level. +CLASSID METADATA + FPA STR fpa +END + +DVO.CAMERADIR STR ctio_mosaic2 # Camera directory for DVO + +# convert supplied FPA.OBSTYPE values to abstract exptype names +OBSTYPE.TABLE METADATA + bias STR BIAS + zero STR BIAS + dark STR DARK + flat STR SKYFLAT + skyflat STR SKYFLAT + domeflat STR DOMEFLAT + object STR OBJECT + science STR OBJECT +END + +# Recipe options +RECIPES METADATA + # Recipes for ppImage + PPIMAGE STR ctio_mosaic2/ppImage.config # Default: all (normal) options on + PSPHOT STR ctio_mosaic2/psphot.config # Default: all (normal) options on + PPMERGE STR ctio_mosaic2/ppMerge.config +END + +# Reduction classes +REDUCTION METADATA + # Detrend processing + DETREND METADATA + BIAS_PROCESS STR PPIMAGE_O + BIAS_RESID STR PPIMAGE_B + BIAS_VERIFY STR PPIMAGE_OB + BIAS_STACK STR PPMERGE_BIAS + DARK_PROCESS STR PPIMAGE_OB + DARK_RESID STR PPIMAGE_D + DARK_VERIFY STR PPIMAGE_OBD + DARK_STACK STR PPMERGE_DARK + SHUTTER_PROCESS STR PPIMAGE_OBD + SHUTTER_RESID STR PPIMAGE_S + SHUTTER_VERIFY STR PPIMAGE_OBDS + SHUTTER_STACK STR PPMERGE_SHUTTER + FLAT_PROCESS STR PPIMAGE_OBDS + FLAT_RESID STR PPIMAGE_F + FLAT_VERIFY STR PPIMAGE_OBDSF + FLAT_STACK STR PPMERGE_FLAT + FRINGE_PROCESS STR PPIMAGE_OBDSF + FRINGE_RESID STR PPIMAGE_R + FRINGE_VERIFY STR PPIMAGE_OBDSFR + FRINGE_STACK STR PPMERGE_FRINGE + + # Generation of pixel masks from darks and flats + DARKMASK_PROCESS STR PPIMAGE_OBD + DARKMASK_RESID STR PPIMAGE_N + DARKMASK_VERIFY STR PPIMAGE_OBD + DARKMASK_STACK STR PPMERGE_DARKMASK + FLATMASK_PROCESS STR PPIMAGE_OBDSF + FLATMASK_RESID STR PPIMAGE_N + FLATMASK_VERIFY STR PPIMAGE_OBDSF + FLATMASK_STACK STR PPMERGE_FLATMASK + JPEG_BIN1_IMAGE_DARKMASK STR PPIMAGE_J1_IMAGE_M + JPEG_BIN2_IMAGE_DARKMASK STR PPIMAGE_J2_IMAGE_M + JPEG_BIN1_IMAGE_FLATMASK STR PPIMAGE_J1_IMAGE_M + JPEG_BIN2_IMAGE_FLATMASK STR PPIMAGE_J2_IMAGE_M + JPEG_BIN1_RESID_DARKMASK STR PPIMAGE_J1_RESID_M + JPEG_BIN2_RESID_DARKMASK STR PPIMAGE_J2_RESID_M + JPEG_BIN1_RESID_FLATMASK STR PPIMAGE_J1_RESID_M + JPEG_BIN2_RESID_FLATMASK STR PPIMAGE_J2_RESID_M + + JPEG_BIN1_IMAGE_BIAS STR PPIMAGE_J1_IMAGE_B + JPEG_BIN1_IMAGE_DARK STR PPIMAGE_J1_IMAGE_B + JPEG_BIN1_IMAGE_SHUTTER STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_FLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_DOMEFLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_SKYFLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_FRINGE STR PPIMAGE_J1_IMAGE_R + JPEG_BIN2_IMAGE_BIAS STR PPIMAGE_J2_IMAGE_B + JPEG_BIN2_IMAGE_DARK STR PPIMAGE_J2_IMAGE_B + JPEG_BIN2_IMAGE_SHUTTER STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_FLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_DOMEFLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_SKYFLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_FRINGE STR PPIMAGE_J2_IMAGE_R + + JPEG_BIN1_RESID_BIAS STR PPIMAGE_J1_RESID_B + JPEG_BIN1_RESID_DARK STR PPIMAGE_J1_RESID_B + JPEG_BIN1_RESID_SHUTTER STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_FLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_DOMEFLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_SKYFLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_FRINGE STR PPIMAGE_J1_RESID_R + JPEG_BIN2_RESID_BIAS STR PPIMAGE_J2_RESID_B + JPEG_BIN2_RESID_DARK STR PPIMAGE_J2_RESID_B + JPEG_BIN2_RESID_SHUTTER STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_FLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_DOMEFLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_SKYFLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_FRINGE STR PPIMAGE_J2_RESID_R + END + # Processing raw data + DEFAULT METADATA + CHIP STR PPIMAGE_OBDSFRA + END +END + +FITS METADATA +# BITPIX is the bits per pixel for writing the output data +# COMP = NONE|RICE|GZIP|HCOMPRESS|PLIO is the compression algorithm +# TILE.[XYZ] are the tile sizes. 0 means entire the dimension, so (0,1,1) forms tiles from rows +# NOISE [0..16] is the number of "noise bits" to preserve when quantising floating point data; 16 for no loss +# HSCALE is the scale factor for lossy compression with HCOMPRESS; 0 or 1 for none; 2*RMS --> 10x compression +# HSMOOTH is the smoothing to apply to HCOMPRESSed data when decompressing; 0 for none + +# BITPIX(S32) is the bits per pixel for writing the output data +# COMPRESSION(STR) = NONE|RICE|GZIP|HCOMPRESS|PLIO is the compression algorithm +# TILE.[XYZ](S32) are the tile sizes. 0 means entire the dimension, so (0,1,1) forms tiles from rows +# NOISE(S32) [0..16] is the number of "noise bits" to preserve when quantising floating point data +# HSCALE(S32) is the scale factor for lossy compression with HCOMPRESS; 0 or 1 for none; 2*RMS --> 10x compression +# HSMOOTH(S32) is the smoothing to apply to HCOMPRESSed data when decompressing; 0 for none +# SCALING(STR) = NONE|RANGE|STDEV_POSITIVE|STDEV_NEGATIVE|STDEV_BOTH|MANUAL is the scaling scheme +# BSCALE(F32) is the manual scaling to apply (when SCALING = MANUAL) +# BZERO(F32) is the manual zero-point to apply (when SCALING = MANUAL) +# STDEV.BITS(S32) is the number of bits to map to a standard deviation (when SCALING = STDEV_*) +# STDEV.NUM(F32) is the number of standard deviations to the edge (when SCALING = STDEV_NEGATIVE|STDEV_POSITIVE) +# FLOAT(STR) is the name of a custom floating-point type + + DET_IMAGE METADATA + BITPIX S32 -32 + END + DET_MASK METADATA + BITPIX S32 8 + END + DET_WEIGHT METADATA + BITPIX S32 -32 + END + + SKY_IMAGE METADATA + BITPIX S32 -32 + END + SKY_MASK METADATA + BITPIX S32 8 + END + SKY_WEIGHT METADATA + BITPIX S32 -32 + END + + COMP_POS METADATA + BITPIX S32 16 + SCALING STR STDEV_POSITIVE + STDEV.BITS S32 4 + STDEV.NUM F32 10 + COMPRESSION STR RICE + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + COMP_MASK METADATA + BITPIX S32 8 + COMPRESSION STR PLIO + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + COMP_SUB METADATA + BITPIX S32 16 + SCALING STR STDEV_BOTH + STDEV.BITS S32 4 + STDEV.NUM F32 5 + COMPRESSION STR RICE + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + +END + +FILERULES METADATA + ### Redirections + PSASTRO.INPUT STR PSASTRO.INPUT.CMF + PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.MEF + PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF + PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.MEF + PPIMAGE.OUTPUT STR PPIMAGE.OUTPUT.MEF + PPIMAGE.BIN1 STR PPIMAGE.BIN1.MEF + PPIMAGE.BIN2 STR PPIMAGE.BIN2.MEF + DVOCORR.OUTPUT STR DVOCORR.MEF.OUTPUT + DVOFLAT.OUTPUT STR DVOFLAT.MEF.OUTPUT + PPIMAGE.OUTPUT.MASK STR PPIMAGE.OUT.MK.MEF + PPIMAGE.OUTPUT.WEIGHT STR PPIMAGE.OUT.WT.MEF + + ### input file definitions + ### use @DETDB entries to get the detrend images from the database + ### replace @DETDB with @FILES if you want to require it from the + ### command line, or with an explicit name to require a specific file + TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + + ## files used by ppImage + PPIMAGE.INPUT INPUT @FILES CHIP IMAGE + PPIMAGE.MASK INPUT @DETDB CHIP IMAGE + PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE + PPIMAGE.DARK INPUT @DETDB CHIP DARK + PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE + PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE + PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE + + ## Files used by ppMerge + PPMERGE.INPUT INPUT @FILES CHIP IMAGE + PPMERGE.INPUT.MASK INPUT @FILES CHIP MASK + PPMERGE.INPUT.WEIGHT INPUT @FILES CHIP WEIGHT + + ## files used to build and apply the flat-field correction images + DVOCORR.INPUT INPUT @FILES CHIP IMAGE + DVOCORR.REFHEAD INPUT @FILES CHIP HEADER + DVOFLAT.INPUT INPUT @FILES CHIP IMAGE + DVOFLAT.CORR INPUT @DETDB CHIP IMAGE + + ## files used by psphot + PSPHOT.LOAD INPUT @FILES CHIP IMAGE + PSPHOT.INPUT INPUT @FILES CHIP IMAGE + PSPHOT.MASK INPUT @FILES CHIP MASK + PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT + PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF + PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + + ## files used by psastro + PSASTRO.INPUT.CMP INPUT @FILES CHIP CMP + PSASTRO.INPUT.CMF INPUT @FILES CHIP CMF + + ## files used by pswarp + PSWARP.INPUT INPUT @FILES CHIP IMAGE + PSWARP.WEIGHT INPUT @FILES CHIP WEIGHT + PSWARP.MASK INPUT @FILES CHIP MASK + PSWARP.SKYCELL INPUT @FILES CHIP IMAGE + PSWARP.ASTROM INPUT @FILES CHIP CMF + + PPSUB.INPUT INPUT @FILES FPA IMAGE + PPSUB.INPUT.MASK INPUT @FILES FPA MASK + PPSUB.INPUT.WEIGHT INPUT @FILES FPA WEIGHT + PPSUB.REF INPUT @FILES FPA IMAGE + PPSUB.REF.MASK INPUT @FILES FPA MASK + PPSUB.REF.WEIGHT INPUT @FILES FPA WEIGHT + PPSUB.SOURCES INPUT @FILES FPA CMF + + PPSTACK.INPUT INPUT @FILES FPA IMAGE + PPSTACK.INPUT.MASK INPUT @FILES FPA MASK + PPSTACK.INPUT.WEIGHT INPUT @FILES FPA WEIGHT + PPSTACK.SOURCES INPUT @FILES FPA CMF + + PPSTAMP.INPUT INPUT @FILES CHIP IMAGE + + PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE + PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + + ### output file definitions + TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT + PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + PPIMAGE.OUTPUT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b0.fits IMAGE NONE CHIP TRUE SPLIT + PPIMAGE.OUTPUT.MEF OUTPUT {OUTPUT}.b0.fits IMAGE COMP_SUB CHIP TRUE MEF + #PPIMAGE.OUTPUT.DETMASK OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE COMP_MASK CHIP TRUE NONE + PPIMAGE.OUTPUT.DETMASK OUTPUT {OUTPUT}.detmask.fits IMAGE COMP_MASK CHIP TRUE MEF + + PPIMAGE.OUT.MK.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits MASK COMP_MASK CHIP TRUE SPLIT + PPIMAGE.OUT.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT COMP_POS CHIP TRUE SPLIT + PPIMAGE.OUT.MK.MEF OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK CHIP TRUE MEF + PPIMAGE.OUT.WT.MEF OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_POS CHIP TRUE MEF + + PPIMAGE.CHIP OUTPUT {OUTPUT}.fits IMAGE COMP_POS CHIP TRUE MEF + PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK CHIP TRUE MEF + PPIMAGE.CHIP.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_POS CHIP TRUE MEF + + PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE MEF + PPIMAGE.OUTPUT.FPA2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE MEF + + PPIMAGE.BIN1.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE NONE CHIP TRUE SPLIT + PPIMAGE.BIN2.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE NONE CHIP TRUE SPLIT + PPIMAGE.BIN1.MEF OUTPUT {OUTPUT}.b1c.fits IMAGE NONE CHIP TRUE MEF + PPIMAGE.BIN2.MEF OUTPUT {OUTPUT}.b2c.fits IMAGE NONE CHIP TRUE MEF + + PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE + PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE + + PPMERGE.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE + PPMERGE.OUTPUT.BIAS OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE + PPMERGE.OUTPUT.DARK OUTPUT {OUTPUT}.fits DARK NONE CHIP TRUE NONE + PPMERGE.OUTPUT.SHUTTER OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE + PPMERGE.OUTPUT.FLAT OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE + PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.fits FRINGE NONE CHIP TRUE NONE + PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.sigma.fits IMAGE NONE CHIP TRUE NONE + PPMERGE.OUTPUT.COUNT OUTPUT {OUTPUT}.count.fits IMAGE NONE CHIP TRUE NONE + + DVOCORR.MEF.OUTPUT OUTPUT {OUTPUT}.fc.fits IMAGE NONE CHIP TRUE MEF + DVOCORR.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE NONE CHIP TRUE SPLIT + DVOFLAT.MEF.OUTPUT OUTPUT {OUTPUT}.co.fits IMAGE NONE CHIP TRUE MEF + DVOFLAT.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE NONE CHIP TRUE SPLIT + + PSPHOT.RESID OUTPUT {OUTPUT}.res.fits IMAGE NONE CHIP TRUE MEF + PSPHOT.BACKGND OUTPUT {OUTPUT}.bck.fits IMAGE NONE CHIP TRUE MEF + PSPHOT.BACKSUB OUTPUT {OUTPUT}.sub.fits IMAGE NONE CHIP TRUE MEF + PSPHOT.BACKMDL OUTPUT {OUTPUT}.mdl.fits IMAGE NONE CHIP TRUE MEF + + PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW NONE CHIP TRUE NONE + PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.sx SX NONE CHIP TRUE NONE + PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.obj OBJ NONE CHIP TRUE NONE + PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.cmp CMP NONE CHIP TRUE NONE + PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF NONE CHIP TRUE NONE + PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE + + PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE CHIP TRUE MEF + + SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.mnt.png KAPA NONE FPA TRUE NONE + SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.psf.png KAPA NONE FPA TRUE NONE + SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.dap.png KAPA NONE FPA TRUE NONE + + PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE + PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE NONE + PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE NONE + + PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK NONE FPA TRUE NONE + PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE + PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + + SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + + PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK NONE FPA TRUE NONE + PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + + PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK NONE FPA TRUE NONE + PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + + PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + + PPSIM.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT + + PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE COMP_POS CHIP TRUE NONE + PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK COMP_MASK CHIP TRUE NONE + + LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE + LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE + TRACE.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.trace TEXT NONE CHIP TRUE NONE +END + +EXTNAME.RULES METADATA + CMF.HEAD STR {CHIP.NAME}.hdr + CMF.DATA STR {CHIP.NAME}.psf # use .PSF and .EXT? + + PSF.HEAD STR {CHIP.NAME}.hdr + PSF.TABLE STR {CHIP.NAME}.psf_model + PSF.RESID STR {CHIP.NAME}.psf_resid +END + +BLANK.HEADERS METADATA + FPA.TIME STR MJD-OBS + FPA.EXPOSURE STR EXPTIME + FPA.AIRMASS STR AIRMASS +END Index: /tags/pap_merge_030828/ippconfig/esowfi/filerules-mef.mdc =================================================================== --- /tags/pap_merge_030828/ippconfig/esowfi/filerules-mef.mdc (revision 17232) +++ /tags/pap_merge_030828/ippconfig/esowfi/filerules-mef.mdc (revision 17232) @@ -0,0 +1,142 @@ +### Redirections +PSASTRO.INPUT STR PSASTRO.INPUT.CMF +PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.SPL +PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF +PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.SPL + +### input file definitions +### use @DETDB entries to get the detrend images from the database +### replace @DETDB with @FILES if you want to require it from the +### command line, or with an explicit name to require a specific file +TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + +## files used by ppImage +PPIMAGE.INPUT INPUT @FILES CHIP IMAGE +PPIMAGE.MASK INPUT @DETDB CHIP IMAGE +PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE +PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE +PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE +PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE + +## Files used by ppMerge +PPMERGE.INPUT INPUT @FILES CHIP IMAGE +PPMERGE.INPUT.MASK INPUT @FILES CHIP MASK +PPMERGE.INPUT.WEIGHT INPUT @FILES CHIP WEIGHT + +## files used to build and apply the flat field correction images +DVOCORR.INPUT INPUT @FILES CHIP IMAGE +DVOCORR.REFHEAD INPUT @FILES CHIP HEADER +DVOFLAT.INPUT INPUT @FILES CHIP IMAGE +DVOFLAT.CORR INPUT @DETDB CHIP IMAGE + +## files used by psphot +PSPHOT.LOAD INPUT @FILES CHIP IMAGE +PSPHOT.INPUT INPUT @FILES CHIP IMAGE +PSPHOT.MASK INPUT @FILES CHIP MASK +PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT +PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF +PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by psastro +PSASTRO.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by pswarp +PSWARP.INPUT INPUT @FILES CHIP IMAGE +PSWARP.SKYCELL INPUT @FILES CHIP IMAGE +PSWARP.WEIGHT INPUT @FILES CHIP WEIGHT +PSWARP.MASK INPUT @FILES CHIP MASK +PSWARP.ASTROM INPUT @FILES CHIP CMF + +PPSUB.INPUT INPUT none.fits CHIP IMAGE +PPSUB.INPUT.MASK INPUT none.fits CHIP MASK +PPSUB.INPUT.WEIGHT INPUT none.fits CHIP WEIGHT +PPSUB.REF INPUT none.fits CHIP IMAGE +PPSUB.REF.MASK INPUT none.fits CHIP MASK +PPSUB.REF.WEIGHT INPUT none.fits CHIP WEIGHT + +PPSTACK.INPUT INPUT none.fits CHIP IMAGE +PPSTACK.INPUT.MASK INPUT none.fits FPA MASK + +PPSTAMP.INPUT INPUT @FILES CHIP IMAGE + +PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE +PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + +### output file definitions +TYPE OUTPUT FILENAME.RULE FILE.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT +PPIMAGE.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE CHIP TRUE SPLIT +PPIMAGE.OUTPUT.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.mask.fits MASK CHIP TRUE SPLIT +PPIMAGE.OUTPUT.WEIGHT OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT CHIP TRUE SPLIT +PPIMAGE.CHIP OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE CHIP TRUE SPLIT +PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.ch.mask.fits MASK CHIP TRUE SPLIT +PPIMAGE.CHIP.WEIGHT OUTPUT {OUTPUT}.{CHIP.NAME}.ch.wt.fits WEIGHT CHIP TRUE SPLIT +PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.fpa1.fits IMAGE FPA TRUE TOGETHER +PPIMAGE.OUTPUT.FPA2 OUTPUT {OUTPUT}.fpa2.fits IMAGE FPA TRUE TOGETHER +PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS FPA TRUE TOGETHER + +PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG FPA TRUE TOGETHER +PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG FPA TRUE TOGETHER +PPIMAGE.BIN1 OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE CHIP TRUE SPLIT +PPIMAGE.BIN2 OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE CHIP TRUE SPLIT + +PPMERGE.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK CHIP TRUE NONE +PPMERGE.OUTPUT.BIAS OUTPUT {OUTPUT}.fits IMAGE CHIP TRUE NONE +PPMERGE.OUTPUT.DARK OUTPUT {OUTPUT}.fits DARK CHIP TRUE NONE +PPMERGE.OUTPUT.SHUTTER OUTPUT {OUTPUT}.fits IMAGE CHIP TRUE NONE +PPMERGE.OUTPUT.FLAT OUTPUT {OUTPUT}.fits IMAGE CHIP TRUE NONE +PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.fits FRINGE CHIP TRUE NONE +PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.sigma.fits IMAGE CHIP TRUE NONE +PPMERGE.OUTPUT.COUNT OUTPUT {OUTPUT}.count.fits IMAGE CHIP TRUE NONE + +DVOCORR.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE CHIP TRUE SPLIT +DVOFLAT.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE CHIP TRUE SPLIT + +PSPHOT.RESID OUTPUT {OUTPUT}.{CHIP.NAME}.res.fits IMAGE CHIP TRUE SPLIT +PSPHOT.BACKGND OUTPUT {OUTPUT}.{CHIP.NAME}.bck.fits IMAGE CHIP TRUE SPLIT +PSPHOT.BACKSUB OUTPUT {OUTPUT}.{CHIP.NAME}.sub.fits IMAGE CHIP TRUE SPLIT +PSPHOT.BACKMDL OUTPUT {OUTPUT}.{CHIP.NAME}.mdl.fits IMAGE CHIP TRUE SPLIT +PSPHOT.BACKMDL.STDEV OUTPUT {OUTPUT}.{CHIP.NAME}.mdd.fits IMAGE CHIP TRUE SPLIT + +PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW CHIP TRUE SPLIT +PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.{CHIP.NAME}.sx SX CHIP TRUE SPLIT +PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.{CHIP.NAME}.obj OBJ CHIP TRUE SPLIT +PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF CHIP TRUE SPLIT +PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF FPA TRUE TOGETHER + +PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.{CHIP.NAME}.psf PSF CHIP TRUE SPLIT + +SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.{CHIP.NAME}.mnt.png KAPA CHIP TRUE SPLIT +SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.{CHIP.NAME}.psf.png KAPA CHIP TRUE SPLIT +SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.{CHIP.NAME}.dap.png KAPA CHIP TRUE SPLIT + +PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP CHIP TRUE SPLIT +PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF CHIP TRUE SPLIT +PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF FPA TRUE TOGETHER + +PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE FPA TRUE TOGETHER +PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK FPA TRUE TOGETHER +PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT FPA TRUE TOGETHER +PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE FPA TRUE TOGETHER +PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE FPA TRUE TOGETHER + +SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS FPA TRUE NONE +SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL FPA TRUE NONE + +PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE FPA TRUE TOGETHER +PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK FPA TRUE TOGETHER +PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT FPA TRUE TOGETHER + +PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE FPA TRUE TOGETHER +PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK FPA TRUE TOGETHER + +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + +PPSIM.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE CHIP TRUE SPLIT + +PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE + +LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT CHIP TRUE SPLIT +LOG.EXP OUTPUT {OUTPUT}.log TEXT FPA TRUE TOGETHER Index: /tags/pap_merge_030828/ippconfig/gpc1/filerules.mdc =================================================================== --- /tags/pap_merge_030828/ippconfig/gpc1/filerules.mdc (revision 17232) +++ /tags/pap_merge_030828/ippconfig/gpc1/filerules.mdc (revision 17232) @@ -0,0 +1,150 @@ +PSASTRO.INPUT STR PSASTRO.INPUT.CMF +PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.SPL +PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF +PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.SPL + +### input file definitions +TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE +PPIMAGE.INPUT INPUT none.fits CHIP IMAGE +INPUT.MASK INPUT none.fits CHIP MASK +INPUT.WEIGHT INPUT none.fits CHIP WEIGHT +INPUT.PSF INPUT none.fits CHIP PSF +INPUT.SRC INPUT none.fits CHIP CMF +PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE +PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE +PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE +PPIMAGE.MASK INPUT @DETDB CHIP MASK + +## Files used by ppMerge +PPMERGE.INPUT INPUT @FILES CHIP IMAGE +PPMERGE.INPUT.MASK INPUT @FILES CHIP MASK +PPMERGE.INPUT.WEIGHT INPUT @FILES CHIP WEIGHT + +## files used by psphot +PSPHOT.LOAD INPUT @FILES CHIP IMAGE +PSPHOT.INPUT INPUT @FILES CHIP IMAGE +PSPHOT.MASK INPUT @FILES CHIP MASK +PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT +PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF +PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + +PSWARP.INPUT INPUT none.fits CHIP IMAGE +PSWARP.WEIGHT INPUT none.fits CHIP WEIGHT +PSWARP.MASK INPUT none.fits CHIP MASK +PSWARP.SKYCELL INPUT none.fits FPA IMAGE +PSWARP.ASTROM INPUT none.fits CHIP CMF + +PSASTRO.WCS INPUT none.fits CHIP CMF +PSASTRO.MODEL INPUT @DETDB FPA ASTROM + +PSASTRO.INPUT.CMP INPUT none.fits CHIP CMP +PSASTRO.INPUT.CMF INPUT none.fits CHIP CMF + +PPSUB.INPUT INPUT none.fits FPA IMAGE +PPSUB.INPUT.MASK INPUT none.fits FPA MASK +PPSUB.INPUT.WEIGHT INPUT none.fits FPA WEIGHT +PPSUB.REF INPUT none.fits FPA IMAGE +PPSUB.REF.MASK INPUT none.fits FPA MASK +PPSUB.REF.WEIGHT INPUT none.fits FPA WEIGHT + +PPSTACK.INPUT INPUT @FILES FPA IMAGE +PPSTACK.INPUT.MASK INPUT @FILES FPA MASK +PPSTACK.INPUT.WEIGHT INPUT @FILES FPA WEIGHT +PPSTACK.SOURCES INPUT @FILES FPA CMF + +PPSTAMP.INPUT INPUT none.fits CHIP IMAGE + +PPARITH.INPUT.IMAGE INPUT none.fits CHIP IMAGE +PPARITH.INPUT.MASK INPUT none.fits CHIP MASK + +### output file definitions +TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT +PPIMAGE.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE COMP_POS CHIP TRUE NONE +PPIMAGE.OUTPUT.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits MASK COMP_MASK CHIP TRUE NONE +PPIMAGE.OUTPUT.WEIGHT OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT COMP_POS CHIP TRUE NONE +PPIMAGE.OUTPUT.DETMASK OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE COMP_MASK CHIP TRUE NONE + +PPIMAGE.CHIP.RAW OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE COMP_POS CHIP TRUE NONE +# PPIMAGE.CHIP.MK OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE COMP_MASK CHIP TRUE NONE +PPIMAGE.CHIP OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE COMP_POS CHIP TRUE NONE +PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.ch.mk.fits MASK COMP_MASK CHIP TRUE NONE +PPIMAGE.CHIP.WEIGHT OUTPUT {OUTPUT}.{CHIP.NAME}.ch.wt.fits WEIGHT COMP_POS CHIP TRUE NONE + +PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.OUTPUT.FPA2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + +PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.BIN1 OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE NONE CHIP TRUE NONE +PPIMAGE.BIN2 OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE NONE CHIP TRUE NONE + +PPIMAGE.STATS OUTPUT {OUTPUT}.{CHIP.NAME}.stats STATS NONE CHIP TRUE NONE + +PPMERGE.OUTPUT.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.fits MASK NONE CHIP TRUE NONE +PPMERGE.OUTPUT.BIAS OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.DARK OUTPUT {OUTPUT}.{CHIP.NAME}.fits DARK NONE CHIP TRUE NONE +PPMERGE.OUTPUT.SHUTTER OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.FLAT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.{CHIP.NAME}.fits FRINGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.sigma.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.COUNT OUTPUT {OUTPUT}.count.fits IMAGE NONE CHIP TRUE NONE + +DVOCORR.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE NONE CHIP TRUE NONE +DVOFLAT.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE NONE CHIP TRUE NONE + +PSPHOT.RESID OUTPUT {OUTPUT}.{CHIP.NAME}.res.fits IMAGE COMP_SUB CHIP TRUE NONE +PSPHOT.BACKGND OUTPUT {OUTPUT}.{CHIP.NAME}.bck.fits IMAGE COMP_POS CHIP TRUE NONE +PSPHOT.BACKSUB OUTPUT {OUTPUT}.{CHIP.NAME}.sub.fits IMAGE COMP_SUB CHIP TRUE NONE +PSPHOT.BACKMDL OUTPUT {OUTPUT}.{CHIP.NAME}.mdl.fits IMAGE COMP_POS CHIP TRUE NONE + +PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW NONE CHIP TRUE NONE +PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.{CHIP.NAME}.sx SX NONE CHIP TRUE NONE +PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.{CHIP.NAME}.obj OBJ NONE CHIP TRUE NONE +PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.cmp CMP NONE CHIP TRUE NONE +PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF NONE CHIP TRUE NONE +PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE MEF + +PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE CHIP TRUE MEF + +SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.{CHIP.NAME}.mnt.png KAPA NONE CHIP TRUE NONE +SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.{CHIP.NAME}.psf.png KAPA NONE CHIP TRUE NONE +SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.{CHIP.NAME}.dap.png KAPA NONE CHIP TRUE NONE + +PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE +PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE NONE +PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUT.MODEL OUTPUT {OUTPUT}.asm ASTROM.MODEL NONE FPA TRUE NONE +PSASTRO.OUT.REFSTARS OUTPUT {OUTPUT}.aref.fits ASTROM.REFSTARS NONE FPA TRUE NONE + +PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_POS FPA TRUE NONE +PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK COMP_MASK FPA TRUE NONE +PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_POS FPA TRUE NONE +PSWARP.OUTPUT.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE +PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE +PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + +SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE +SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + +PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_SUB FPA TRUE NONE +PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK COMP_MASK FPA TRUE NONE +PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_POS FPA TRUE NONE + +PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_POS FPA TRUE NONE +PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK COMP_MASK FPA TRUE NONE +PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT COMP_POS FPA TRUE NONE + +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.CHIP OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + +PPSIM.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE MEF + +PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE COMP_POS CHIP TRUE NONE +PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK COMP_MASK CHIP TRUE NONE + +LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE +LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE + +TRACE.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.trace TEXT NONE CHIP TRUE NONE +TRACE.EXP OUTPUT {OUTPUT}.trace TEXT NONE FPA TRUE NONE Index: /tags/pap_merge_030828/ippconfig/isp/camera.config =================================================================== --- /tags/pap_merge_030828/ippconfig/isp/camera.config (revision 17232) +++ /tags/pap_merge_030828/ippconfig/isp/camera.config (revision 17232) @@ -0,0 +1,330 @@ +# Camera configuration file for the Pan-STARRS Imaging Sky Probe + +# File formats that we know about +FORMATS METADATA + ISP STR isp/format.config + CMP STR isp/cmp.config + CMF STR isp/cmf.config +END + +# Description of camera --- all the chips and the cells that comprise them +FPA METADATA + Chip STR Cell +END + +# valid filter names and corresponding IDs +FILTER.ID METADATA + g STR g + r STR r + i STR i + z STR z + y STR y +END + +# Table of strings to use for the class identifier (see ippdb) according to the file level. +CLASSID METADATA + FPA STR fpa +END + +DVO.CAMERADIR STR isp # Camera directory for DVO + +# convert supplied FPA.OBSTYPE values to abstract exptype names +OBSTYPE.TABLE METADATA + bias STR BIAS + zero STR BIAS + dark STR DARK + flat STR SKYFLAT + skyflat STR SKYFLAT + domeflat STR DOMEFLAT + object STR OBJECT + science STR OBJECT +END + +# Recipe options +RECIPES METADATA + PSPHOT STR isp/psphot.config # psphot details + PSASTRO STR isp/psastro.config # psastro details + PPIMAGE STR isp/ppImage.config # Recipes for ppImage + PPMERGE STR isp/ppMerge.config # Recipes for ppMerge + REJECTIONS STR isp/rejections.config +END + +# Reduction classes +REDUCTION METADATA + # Detrend processing + DETREND METADATA + BIAS_PROCESS STR PPIMAGE_O + BIAS_RESID STR PPIMAGE_B + BIAS_VERIFY STR PPIMAGE_OB + BIAS_STACK STR PPMERGE_BIAS + DARK_PROCESS STR PPIMAGE_OB + DARK_RESID STR PPIMAGE_D + DARK_VERIFY STR PPIMAGE_OBD + DARK_STACK STR PPMERGE_DARK + SHUTTER_PROCESS STR PPIMAGE_OBD + SHUTTER_RESID STR PPIMAGE_S + SHUTTER_VERIFY STR PPIMAGE_OBDS + SHUTTER_STACK STR PPMERGE_SHUTTER + FLAT_PROCESS STR PPIMAGE_OBDS + FLAT_RESID STR PPIMAGE_F + FLAT_VERIFY STR PPIMAGE_OBDSF + FLAT_STACK STR PPMERGE_FLAT + FRINGE_PROCESS STR PPIMAGE_OBDSF + FRINGE_RESID STR PPIMAGE_R + FRINGE_VERIFY STR PPIMAGE_OBDSFR + FRINGE_STACK STR PPMERGE_FRINGE + + JPEG_BIN1_IMAGE_BIAS STR PPIMAGE_J1_IMAGE_B + JPEG_BIN1_IMAGE_DARK STR PPIMAGE_J1_IMAGE_B + JPEG_BIN1_IMAGE_SHUTTER STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_FLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_DOMEFLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_SKYFLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_FRINGE STR PPIMAGE_J1_IMAGE_R + JPEG_BIN2_IMAGE_BIAS STR PPIMAGE_J2_IMAGE_B + JPEG_BIN2_IMAGE_DARK STR PPIMAGE_J2_IMAGE_B + JPEG_BIN2_IMAGE_SHUTTER STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_FLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_DOMEFLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_SKYFLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_FRINGE STR PPIMAGE_J2_IMAGE_R + + JPEG_BIN1_RESID_BIAS STR PPIMAGE_J1_RESID_B + JPEG_BIN1_RESID_DARK STR PPIMAGE_J1_RESID_B + JPEG_BIN1_RESID_SHUTTER STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_FLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_DOMEFLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_SKYFLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_FRINGE STR PPIMAGE_J1_RESID_R + JPEG_BIN2_RESID_BIAS STR PPIMAGE_J2_RESID_B + JPEG_BIN2_RESID_DARK STR PPIMAGE_J2_RESID_B + JPEG_BIN2_RESID_SHUTTER STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_FLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_DOMEFLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_SKYFLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_FRINGE STR PPIMAGE_J2_RESID_R + END + # Processing raw data + DEFAULT METADATA + CHIP STR PPIMAGE_OBDSFRA + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + END + + # Detrend Processing only for raw data + DETREND_ONLY METADATA + CHIP STR PPIMAGE_DET_ONLY + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + END +END + +FITS METADATA +# BITPIX is the bits per pixel for writing the output data +# COMP = NONE|RICE|GZIP|HCOMPRESS|PLIO is the compression algorithm +# TILE.[XYZ] are the tile sizes. 0 means entire the dimension, so (0,1,1) forms tiles from rows +# NOISE [0..16] is the number of "noise bits" to preserve when quantising floating point data; 16 for no loss +# HSCALE is the scale factor for lossy compression with HCOMPRESS; 0 or 1 for none; 2*RMS --> 10x compression +# HSMOOTH is the smoothing to apply to HCOMPRESSed data when decompressing; 0 for none + +# BITPIX(S32) is the bits per pixel for writing the output data +# COMPRESSION(STR) = NONE|RICE|GZIP|HCOMPRESS|PLIO is the compression algorithm +# TILE.[XYZ](S32) are the tile sizes. 0 means entire the dimension, so (0,1,1) forms tiles from rows +# NOISE(S32) [0..16] is the number of "noise bits" to preserve when quantising floating point data +# HSCALE(S32) is the scale factor for lossy compression with HCOMPRESS; 0 or 1 for none; 2*RMS --> 10x compression +# HSMOOTH(S32) is the smoothing to apply to HCOMPRESSed data when decompressing; 0 for none +# SCALING(STR) = NONE|RANGE|STDEV_POSITIVE|STDEV_NEGATIVE|STDEV_BOTH|MANUAL is the scaling scheme +# BSCALE(F32) is the manual scaling to apply (when SCALING = MANUAL) +# BZERO(F32) is the manual zero-point to apply (when SCALING = MANUAL) +# STDEV.BITS(S32) is the number of bits to map to a standard deviation (when SCALING = STDEV_*) +# STDEV.NUM(F32) is the number of standard deviations to the edge (when SCALING = STDEV_NEGATIVE|STDEV_POSITIVE) +# FLOAT(STR) is the name of a custom floating-point type + + DET_IMAGE METADATA + BITPIX S32 -32 + END + DET_MASK METADATA + BITPIX S32 8 + END + DET_WEIGHT METADATA + BITPIX S32 -32 + END + + SKY_IMAGE METADATA + BITPIX S32 -32 + END + SKY_MASK METADATA + BITPIX S32 8 + END + SKY_WEIGHT METADATA + BITPIX S32 -32 + END + + COMPRESSED_POSITIVE METADATA + BITPIX S32 16 + SCALING STR STDEV_POSITIVE + STDEV.BITS S32 4 + STDEV.NUM F32 10 + COMPRESSION STR RICE + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + COMPRESSED_MASK METADATA + COMPRESSION STR PLIO + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + COMPRESSED_SUBTRACTION METADATA + BITPIX S32 16 + SCALING STR STDEV_BOTH + STDEV.BITS S32 4 + STDEV.NUM F32 5 + COMPRESSION STR RICE + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + +END + +FILERULES METADATA + PSASTRO.INPUT STR PSASTRO.INPUT.CMF + PSASTRO.OUTPUT STR PSASTRO.OUTPUT.CMF + PSASTRO.OUTPUT.MEF STR PSASTRO.OUTPUT.CMF + PSPHOT.OUTPUT STR PSPHOT.OUTPUT.CMF + + ### input file definitions + ### use @DETDB entries to get the detrend images from the database + ### replace @DETDB with @FILES if you want to require it from the + ### command line, or with an explicit name to require a specific file + TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + + ## files used by ppImage + PPIMAGE.INPUT INPUT @FILES FPA IMAGE + PPIMAGE.MASK INPUT mask.fits FPA IMAGE + PPIMAGE.BIAS INPUT @DETDB FPA IMAGE + PPIMAGE.DARK INPUT @DETDB FPA DARK + PPIMAGE.FLAT INPUT @DETDB FPA IMAGE + PPIMAGE.FRINGE INPUT @DETDB FPA FRINGE + PPIMAGE.SHUTTER INPUT @DETDB FPA IMAGE + + ## Files used by ppMerge + PPMERGE.INPUT INPUT @FILES CHIP IMAGE + PPMERGE.INPUT.MASK INPUT @FILES CHIP MASK + PPMERGE.INPUT.WEIGHT INPUT @FILES CHIP WEIGHT + + ## files used to build and apply the flat-field correction images + DVOCORR.INPUT INPUT @FILES FPA IMAGE + DVOCORR.REFHEAD INPUT @FILES FPA HEADER + DVOFLAT.INPUT INPUT @FILES FPA IMAGE + DVOFLAT.CORR INPUT @DETDB FPA IMAGE + + ## files used by psphot + PSPHOT.LOAD INPUT @FILES FPA IMAGE + PSPHOT.INPUT INPUT @FILES FPA IMAGE + PSPHOT.MASK INPUT @FILES FPA MASK + PSPHOT.WEIGHT INPUT @FILES FPA WEIGHT + PSPHOT.PSF.LOAD INPUT @FILES FPA PSF + + ## files used by psastro + PSASTRO.INPUT.CMP INPUT @FILES FPA CMP + PSASTRO.INPUT.CMF INPUT @FILES FPA CMF + + ## files used by pswarp + PSWARP.INPUT INPUT @FILES FPA IMAGE + PSWARP.WEIGHT INPUT @FILES FPA WEIGHT + PSWARP.MASK INPUT @FILES FPA MASK + PSWARP.SKYCELL INPUT @FILES FPA IMAGE + PSWARP.ASTROM INPUT @FILES FPA CMF + + PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE + PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + + ### output file definitions + TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT + PPIMAGE.OUTPUT OUTPUT {OUTPUT}.isp.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE + PPIMAGE.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + PPIMAGE.CHIP OUTPUT {OUTPUT}.chip.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.chip.mask.fits MASK NONE FPA TRUE NONE + PPIMAGE.CHIP.WEIGHT OUTPUT {OUTPUT}.chip.wt.fits WEIGHT NONE FPA TRUE NONE + PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.OUTPUT.FPA2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + + PPIMAGE.BIN1 OUTPUT {OUTPUT}.b1c.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.BIN2 OUTPUT {OUTPUT}.b2c.fits IMAGE NONE FPA TRUE NONE + + PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE + PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE + + PPMERGE.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE + PPMERGE.OUTPUT.BIAS OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE + PPMERGE.OUTPUT.DARK OUTPUT {OUTPUT}.fits DARK NONE CHIP TRUE NONE + PPMERGE.OUTPUT.SHUTTER OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE + PPMERGE.OUTPUT.FLAT OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE + PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.fits FRINGE NONE CHIP TRUE NONE + PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.sigma.fits IMAGE NONE CHIP TRUE NONE + PPMERGE.OUTPUT.COUNT OUTPUT {OUTPUT}.count.fits IMAGE NONE CHIP TRUE NONE + + DVOCORR.OUTPUT OUTPUT {OUTPUT}.fc.fits IMAGE NONE FPA TRUE NONE + DVOFLAT.OUTPUT OUTPUT {OUTPUT}.co.fits IMAGE NONE FPA TRUE NONE + + PSPHOT.RESID OUTPUT {OUTPUT}.res.fits IMAGE NONE FPA TRUE NONE + PSPHOT.BACKGND OUTPUT {OUTPUT}.bck.fits IMAGE NONE FPA TRUE NONE + PSPHOT.BACKSUB OUTPUT {OUTPUT}.sub.fits IMAGE NONE FPA TRUE NONE + PSPHOT.BACKMDL OUTPUT {OUTPUT}.mdl.fits IMAGE NONE FPA TRUE NONE + + PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT} RAW NONE FPA TRUE NONE + PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.sx SX NONE FPA TRUE NONE + PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.obj OBJ NONE FPA TRUE NONE + PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.cmp CMP NONE FPA TRUE NONE + PSPHOT.OUTPUT.CMF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE + + PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE FPA TRUE NONE + + SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.mnt.png KAPA NONE FPA TRUE NONE + SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.psf.png KAPA NONE FPA TRUE NONE + SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.dap.png KAPA NONE FPA TRUE NONE + + PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.smp CMP NONE FPA TRUE NONE + PSASTRO.OUTPUT.CMF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE NONE + + PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE + PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE + PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + + SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + + PPSIM.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE ISP + PPSIM.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE + + PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE + PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE + + LOG.IMFILE OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE + LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE +END + +EXTNAME.RULES METADATA + CMF.HEAD STR cmf.hdr + CMF.DATA STR cmf.psf # use .PSF and .EXT? + + PSF.HEAD STR hdr + PSF.TABLE STR psf_model + PSF.RESID STR psf_resid +END + +BLANK.HEADERS METADATA + FPA.TIME STR MJD-OBS + FPA.EXPOSURE STR EXPTIME + FPA.AIRMASS STR AIRMASS +END Index: /tags/pap_merge_030828/ippconfig/megacam/filerules-mef.mdc =================================================================== --- /tags/pap_merge_030828/ippconfig/megacam/filerules-mef.mdc (revision 17232) +++ /tags/pap_merge_030828/ippconfig/megacam/filerules-mef.mdc (revision 17232) @@ -0,0 +1,195 @@ +### File rules for PHU=FPA, EXT=CHIP + +### Redirections +PPIMAGE.OUTPUT STR PPIMAGE.OUTPUT.MEF +PPIMAGE.OUTPUT.MASK STR PPIMAGE.OUT.MK.MEF +PPIMAGE.OUTPUT.WEIGHT STR PPIMAGE.OUT.WT.MEF + +PPIMAGE.CHIP STR PPIMAGE.CHIP.MEF +PPIMAGE.CHIP.MASK STR PPIMAGE.CHIP.MK.MEF +PPIMAGE.CHIP.WEIGHT STR PPIMAGE.CHIP.WT.MEF + +PPIMAGE.OUTPUT.FPA1 STR PPIMAGE.OUTPUT.FPA1.MEF +PPIMAGE.OUTPUT.FPA2 STR PPIMAGE.OUTPUT.FPA2.MEF +PPIMAGE.BIN1 STR PPIMAGE.BIN1.MEF +PPIMAGE.BIN2 STR PPIMAGE.BIN2.MEF + +PPSTAMP.CHIP STR PPSTAMP.CHIP.MEF + +PSASTRO.INPUT STR PSASTRO.INPUT.CMF +PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.MEF +PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF +PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.MEF + +DVOCORR.OUTPUT STR DVOCORR.MEF.OUTPUT +DVOFLAT.OUTPUT STR DVOFLAT.MEF.OUTPUT + +### input file definitions +### use @DETDB entries to get the detrend images from the database +### replace @DETDB with @FILES if you want to require it from the +### command line, or with an explicit name to require a specific file +TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + +## files used by ppImage +PPIMAGE.INPUT INPUT @FILES CHIP IMAGE +PPIMAGE.MASK INPUT @DETDB CHIP MASK +PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE +PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE +PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE +PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE + +## Files used by ppMerge +PPMERGE.INPUT INPUT @FILES CHIP IMAGE +PPMERGE.INPUT.MASK INPUT @FILES CHIP MASK +PPMERGE.INPUT.WEIGHT INPUT @FILES CHIP WEIGHT + +## files used to build and apply the flat-field correction images +DVOCORR.INPUT INPUT @FILES CHIP IMAGE +DVOCORR.REFHEAD INPUT @FILES CHIP HEADER +DVOFLAT.INPUT INPUT @FILES CHIP IMAGE +DVOFLAT.CORR INPUT @DETDB CHIP IMAGE + +## files used by psphot +PSPHOT.LOAD INPUT @FILES CHIP IMAGE +PSPHOT.INPUT INPUT @FILES CHIP IMAGE +PSPHOT.MASK INPUT @FILES CHIP MASK +PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT +PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF +PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by psastro +PSASTRO.WCS INPUT none.fits CHIP CMF +PSASTRO.MODEL INPUT @DETDB FPA ASTROM +PSASTRO.INPUT.CMP INPUT @FILES CHIP CMP +PSASTRO.INPUT.CMF INPUT @FILES CHIP CMF + +## files used by pswarp +PSWARP.INPUT INPUT @FILES CHIP IMAGE +PSWARP.WEIGHT INPUT @FILES CHIP WEIGHT +PSWARP.MASK INPUT @FILES CHIP MASK +PSWARP.SKYCELL INPUT @FILES CHIP IMAGE +PSWARP.ASTROM INPUT @FILES CHIP CMF + +PPSUB.INPUT INPUT @FILES FPA IMAGE +PPSUB.INPUT.MASK INPUT @FILES FPA MASK +PPSUB.INPUT.WEIGHT INPUT @FILES FPA WEIGHT +PPSUB.REF INPUT @FILES FPA IMAGE +PPSUB.REF.MASK INPUT @FILES FPA MASK +PPSUB.REF.WEIGHT INPUT @FILES FPA WEIGHT +PPSUB.SOURCES INPUT @FILES FPA CMF + +PPSTACK.INPUT INPUT @FILES FPA IMAGE +PPSTACK.INPUT.MASK INPUT @FILES FPA MASK +PPSTACK.INPUT.WEIGHT INPUT @FILES FPA WEIGHT +PPSTACK.INPUT.PSF INPUT @FILES CHIP PSF +PPSTACK.SOURCES INPUT @FILES FPA CMF + +PPSTAMP.INPUT INPUT @FILES CHIP IMAGE + +PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE +PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + +### output file definitions +TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT +PPIMAGE.OUTPUT.MEF OUTPUT {OUTPUT}.b0.fits IMAGE NONE CHIP TRUE MEF +PPIMAGE.OUT.MK.MEF OUTPUT {OUTPUT}.mk.fits MASK NONE CHIP TRUE MEF +PPIMAGE.OUT.WT.MEF OUTPUT {OUTPUT}.wt.fits WEIGHT NONE CHIP TRUE MEF +PPIMAGE.OUTPUT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b0.fits IMAGE NONE CHIP TRUE SPLIT +PPIMAGE.OUT.MK.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits MASK NONE CHIP TRUE SPLIT +PPIMAGE.OUT.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT NONE CHIP TRUE SPLIT +PPIMAGE.OUTPUT.DETMASK OUTPUT {OUTPUT}.fits IMAGE COMP_MASK CHIP TRUE MEF + +PPIMAGE.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP TRUE MEF +PPIMAGE.CHIP.MK.MEF OUTPUT {OUTPUT}.ch.mk.fits MASK NONE CHIP TRUE MEF +PPIMAGE.CHIP.WT.MEF OUTPUT {OUTPUT}.ch.wt.fits WEIGHT NONE CHIP TRUE MEF +PPIMAGE.CHIP.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE NONE CHIP TRUE SPLIT +PPIMAGE.CHIP.MK.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.mk.fits MASK NONE CHIP TRUE SPLIT +PPIMAGE.CHIP.WT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.ch.wt.fits WEIGHT NONE CHIP TRUE SPLIT + +PPIMAGE.OUTPUT.FPA1.MEF OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE MEF +PPIMAGE.OUTPUT.FPA2.MEF OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE MEF +PPIMAGE.OUTPUT.FPA1.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE NONE FPA TRUE SPLIT +PPIMAGE.OUTPUT.FPA2.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE NONE FPA TRUE SPLIT + +PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE MEF + +PPIMAGE.BIN1.MEF OUTPUT {OUTPUT}.b1c.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.BIN2.MEF OUTPUT {OUTPUT}.b2c.fits IMAGE COMP_IMG CHIP TRUE MEF +PPIMAGE.BIN1.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE COMP_IMG CHIP TRUE SPLIT +PPIMAGE.BIN2.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE COMP_IMG CHIP TRUE SPLIT + +PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE + +PPMERGE.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE +PPMERGE.OUTPUT.BIAS OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.DARK OUTPUT {OUTPUT}.fits DARK NONE CHIP TRUE NONE +PPMERGE.OUTPUT.SHUTTER OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.FLAT OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.fits FRINGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.sigma.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.COUNT OUTPUT {OUTPUT}.count.fits IMAGE NONE CHIP TRUE NONE + +DVOCORR.MEF.OUTPUT OUTPUT {OUTPUT}.fc.fits IMAGE NONE CHIP TRUE MEF +DVOCORR.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE NONE CHIP TRUE SPLIT +DVOFLAT.MEF.OUTPUT OUTPUT {OUTPUT}.co.fits IMAGE NONE CHIP TRUE MEF +DVOFLAT.SPL.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE NONE CHIP TRUE SPLIT + +PSPHOT.RESID OUTPUT {OUTPUT}.res.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKGND OUTPUT {OUTPUT}.bck.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKSUB OUTPUT {OUTPUT}.sub.fits IMAGE NONE CHIP FALSE MEF +PSPHOT.BACKMDL OUTPUT {OUTPUT}.mdl.fits IMAGE NONE CHIP FALSE MEF + +PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW NONE CHIP TRUE NONE +PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.{CHIP.NAME}.sx SX NONE CHIP TRUE NONE +PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.{CHIP.NAME}.obj OBJ NONE CHIP TRUE NONE +PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.cmp CMP NONE CHIP TRUE NONE +PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF NONE CHIP TRUE SPLIT +PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF NONE CHIP TRUE MEF + +PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE CHIP TRUE MEF + +SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.mnt.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.psf.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.dap.png KAPA NONE FPA TRUE NONE + +PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE +PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE SPLIT +PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUT.MODEL OUTPUT {OUTPUT}.asm ASTROM.MODEL NONE FPA TRUE NONE +PSASTRO.OUT.REFSTARS OUTPUT {OUTPUT}.aref.fits ASTROM.REFSTARS NONE FPA TRUE NONE + +PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK NONE FPA TRUE NONE +PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE +PSWARP.OUTPUT.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE +PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE +PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + +SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE +SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + +PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK NONE FPA TRUE NONE +PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + +PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK NONE FPA TRUE NONE +PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + +PPSIM.OUTPUT.MEF OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE MEF +PPSIM.OUTPUT.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT +PPSIM.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE MEF + +PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE COMP_IMG CHIP TRUE NONE +PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK COMP_MASK CHIP TRUE NONE + +LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE +LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE + +TRACE.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.trace TEXT NONE CHIP TRUE NONE +TRACE.EXP OUTPUT {OUTPUT}.trace TEXT NONE FPA TRUE NONE Index: /tags/pap_merge_030828/ippconfig/megacam/ppMerge.config =================================================================== --- /tags/pap_merge_030828/ippconfig/megacam/ppMerge.config (revision 17232) +++ /tags/pap_merge_030828/ippconfig/megacam/ppMerge.config (revision 17232) @@ -0,0 +1,40 @@ + +# Bias combination --- don't want min/max rejection +PPMERGE_BIAS METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 2 # Number of rejection iterations + FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately + FRACLOW F32 0.0 # Fraction of low pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR CLIPPED # Statistic to use for combination: +END + +# Dark combination --- don't want min/max rejection +# More aggressive clipping than bias, so as to remove CRs +PPMERGE_DARK METADATA + REJ F32 2.0 # Rejection threshold (sigma) + ITER S32 4 # Number of rejection iterations + FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately + FRACLOW F32 0.0 # Fraction of low pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR CLIPPED # Statistic to use for combination: +END + +# Flat combination --- use min/max rejection +PPMERGE_FLAT METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 1 # Number of rejection iterations + FRACHIGH F32 0.3 # Fraction of high pixels to reject immediately + FRACLOW F32 0.1 # Fraction of low pixels to reject immediately + NKEEP S32 5 # Minimum number of pixels in stack to keep + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR MEAN # Statistic to use for combination: +END + + +# Fringe combination --- already included in default, above +PPMERGE_FRINGE METADATA + FRACHIGH F32 0.1 # Fraction of high pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? +END + Index: /tags/pap_merge_030828/ippconfig/recipes/ppMerge.config =================================================================== --- /tags/pap_merge_030828/ippconfig/recipes/ppMerge.config (revision 17232) +++ /tags/pap_merge_030828/ippconfig/recipes/ppMerge.config (revision 17232) @@ -0,0 +1,95 @@ +# Recipe configuration for ppMerge + +ROWS S32 128 # Number of rows to read at once +ELECTRONS F32 100.0 # Minimum number of electrons for useful signal +SAMPLE S32 100000 # Sampling factor for measuring the background +REJ F32 3.0 # Rejection threshold (sigma) +ITER S32 0 # Number of rejection iterations +FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately +FRACLOW F32 0.0 # Fraction of low pixels to reject immediately +NKEEP S32 5 # Minimum number of pixels in stack to keep +WEIGHTS BOOL FALSE # Use image weights in combination? +FRINGE.NUM S32 10000 # Number of fringe regions +FRINGE.SIZE S32 5 # Half-size of fringe regions +FRINGE.XSMOOTH S32 5 # Number of smoothing regions in x +FRINGE.YSMOOTH S32 11 # Number of smoothing regions in y +SHUTTER.SIZE S32 128 # Size for shutter measurement regions +MASK.SUSPECT F32 5.0 # Threshold for suspect pixels (sigma) +MASK.BAD F32 0.2 # Threshold for bad pixels +MASK.MODE STR FRACTION # Mode for identifying bad pixels in the suspect map +MASK.CHIPSTATS BOOL TRUE # Measure stats for masking by chip (otherwise by readout)? +MASK.GROW S32 0 # Grow bad pixels by this radius +MASK.GROWVAL STR SUSPECT # Give grown mask pixels this value +MASKVAL STR SAT,BAD # Mask value for input data +COMBINE STR CLIPPED # Statistic to use for combination +MEAN STR ROBUST_MEDIAN # Statistic to use to measure the mean +STDEV STR ROBUST_STDEV # Statistic to use to measure the stdev + +STATS.BY.CHIP BOOL TRUE # measure stats for masking by chip (or by readout) +MASK.GROW.NPIX S32 3 # measure stats for masking by chip (or by readout) + +# Ordinates for fitting dark current +DARK.ORDINATES METADATA + CELL.DARKTIME S32 1 # Traditional dark current term +END +DARK.NORM STR NONE # Dark normalisation concept + +# Bias combination --- don't want min/max rejection +PPMERGE_BIAS METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 2 # Number of rejection iterations + FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately + FRACLOW F32 0.0 # Fraction of low pixels to reject immediately + WEIGHTS BOOL FALSE # Use image weights? + COMBINE STR CLIPPED # Statistic to use for combination: +END + + +# Dark combination --- don't want min/max rejection +# More aggressive clipping than bias, so as to remove CRs +PPMERGE_DARK METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 2 # Number of rejection iterations + FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately + FRACLOW F32 0.0 # Fraction of low pixels to reject immediately + WEIGHTS BOOL TRUE # Use image weights? + COMBINE STR CLIPPED # Statistic to use for combination: +END + +# Flat combination --- use min/max rejection +PPMERGE_FLAT METADATA + REJ F32 3.0 # Rejection threshold (sigma) + ITER S32 1 # Number of rejection iterations + FRACHIGH F32 0.3 # Fraction of high pixels to reject immediately + FRACLOW F32 0.1 # Fraction of low pixels to reject immediately + NKEEP S32 5 # Minimum number of pixels in stack to keep + WEIGHTS BOOL TRUE # Use image weights? + COMBINE STR MEAN # Statistic to use for combination: +END + + +# Fringe combination --- already included in default, above +PPMERGE_FRINGE METADATA + FRACHIGH F32 0.1 # Fraction of high pixels to reject immediately + WEIGHTS BOOL TRUE # Use image weights? +END + +# Mask generation --- already included in default, above +PPMERGE_DARKMASK METADATA + ITER S32 2 # Number of iterations + MASK.BAD F32 0.2 # Threshold for bad pixels (sigma) + MASK.MODE STR FRACTION # Mode for identifying bad pixels in the suspect map +END + +# Mask generation --- already included in default, above +PPMERGE_FLATMASK METADATA + ITER S32 2 # Number of iterations + MASK.BAD F32 0.2 # Threshold for bad pixels (sigma) + MASK.MODE STR FRACTION # Mode for identifying bad pixels in the suspect map +END + +# Shutter generation --- already included in default, above +PPMERGE_SHUTTER METADATA + REJ F32 2.0 # Rejection threshold (sigma) + ITER S32 1 # Number of rejection iterations +END Index: /tags/pap_merge_030828/ippconfig/simmosaic/camera.config =================================================================== --- /tags/pap_merge_030828/ippconfig/simmosaic/camera.config (revision 17232) +++ /tags/pap_merge_030828/ippconfig/simmosaic/camera.config (revision 17232) @@ -0,0 +1,388 @@ +# Camera configuration file for simulated mosaic + +# File formats that we know about +FORMATS METADATA + TOGETHER STR simmosaic/format_together.config + SPLIT STR simmosaic/format_split.config +END + +# Description of camera --- all the chips and the cells that comprise them +FPA METADATA + Chip00 STR Cell00 Cell01 Cell10 Cell11 + Chip01 STR Cell00 Cell01 Cell10 Cell11 + Chip10 STR Cell00 Cell01 Cell10 Cell11 + Chip11 STR Cell00 Cell01 Cell10 Cell11 +END + +# move this elsewhere? we need a lookup table to go from filter ID to abstract name +FILTER.ID METADATA + NONE STR NONE + B STR B + V STR V + R STR R + I STR I + g STR g + r STR r + i STR i + z STR z +END + +# Table of strings to use for the class identifier (see ippdb) according to the file level. +CLASSID METADATA + FPA STR fpa + CHIP STR {CHIP.NAME} + CELL STR {CHIP.NAME}.{CELL.NAME} + fpa STR fpa + chip STR {CHIP.NAME} + cell STR {CHIP.NAME}.{CELL.NAME} +END + +DVO.CAMERADIR STR simmosaic # Camera directory for DVO + +# convert supplied FPA.OBSTYPE values to abstract exptype names +OBSTYPE.TABLE METADATA + bias STR BIAS + zero STR BIAS + dark STR DARK + flat STR SKYFLAT + skyflat STR SKYFLAT + domeflat STR DOMEFLAT + object STR OBJECT + science STR OBJECT +END + +# Recipe options +RECIPES METADATA + PPIMAGE STR simmosaic/ppImage.config # Default: all (normal) options on + PSPHOT STR simmosaic/psphot.config # psphot details + PSASTRO STR simmosaic/psastro.config # psastro details + REJECTIONS STR simmosaic/rejections.config # Rejection limits +END + +# Reduction classes +REDUCTION METADATA + # Detrend processing + DETREND METADATA + BIAS_PROCESS STR PPIMAGE_O + BIAS_RESID STR PPIMAGE_B + BIAS_VERIFY STR PPIMAGE_OB + BIAS_STACK STR PPMERGE_BIAS + DARK_PROCESS STR PPIMAGE_OB + DARK_RESID STR PPIMAGE_D + DARK_VERIFY STR PPIMAGE_OBD + DARK_STACK STR PPMERGE_DARK + SHUTTER_PROCESS STR PPIMAGE_OBD + SHUTTER_RESID STR PPIMAGE_S + SHUTTER_VERIFY STR PPIMAGE_OBDS + SHUTTER_STACK STR PPMERGE_SHUTTER + FLAT_PROCESS STR PPIMAGE_OBDS + FLAT_RESID STR PPIMAGE_F + FLAT_VERIFY STR PPIMAGE_OBDSF + FLAT_STACK STR PPMERGE_FLAT + FRINGE_PROCESS STR PPIMAGE_OBDSF + FRINGE_RESID STR PPIMAGE_R + FRINGE_VERIFY STR PPIMAGE_OBDSFR + FRINGE_STACK STR PPMERGE_FRINGE + + # Generation of pixel masks from darks and flats + DARKMASK_PROCESS STR PPIMAGE_OBD + DARKMASK_RESID STR PPIMAGE_N + DARKMASK_VERIFY STR PPIMAGE_OBD + DARKMASK_STACK STR PPMERGE_DARKMASK + FLATMASK_PROCESS STR PPIMAGE_OBDSF + FLATMASK_RESID STR PPIMAGE_N + FLATMASK_VERIFY STR PPIMAGE_OBDSF + FLATMASK_STACK STR PPMERGE_FLATMASK + JPEG_BIN1_IMAGE_DARKMASK STR PPIMAGE_J1_IMAGE_M + JPEG_BIN2_IMAGE_DARKMASK STR PPIMAGE_J2_IMAGE_M + JPEG_BIN1_IMAGE_FLATMASK STR PPIMAGE_J1_IMAGE_M + JPEG_BIN2_IMAGE_FLATMASK STR PPIMAGE_J2_IMAGE_M + JPEG_BIN1_RESID_DARKMASK STR PPIMAGE_J1_RESID_M + JPEG_BIN2_RESID_DARKMASK STR PPIMAGE_J2_RESID_M + JPEG_BIN1_RESID_FLATMASK STR PPIMAGE_J1_RESID_M + JPEG_BIN2_RESID_FLATMASK STR PPIMAGE_J2_RESID_M + + JPEG_BIN1_IMAGE_BIAS STR PPIMAGE_J1_IMAGE_B + JPEG_BIN1_IMAGE_DARK STR PPIMAGE_J1_IMAGE_B + JPEG_BIN1_IMAGE_SHUTTER STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_FLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_DOMEFLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_SKYFLAT STR PPIMAGE_J1_IMAGE_F + JPEG_BIN1_IMAGE_FRINGE STR PPIMAGE_J1_IMAGE_R + JPEG_BIN2_IMAGE_BIAS STR PPIMAGE_J2_IMAGE_B + JPEG_BIN2_IMAGE_DARK STR PPIMAGE_J2_IMAGE_B + JPEG_BIN2_IMAGE_SHUTTER STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_FLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_DOMEFLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_SKYFLAT STR PPIMAGE_J2_IMAGE_F + JPEG_BIN2_IMAGE_FRINGE STR PPIMAGE_J2_IMAGE_R + + JPEG_BIN1_RESID_BIAS STR PPIMAGE_J1_RESID_B + JPEG_BIN1_RESID_DARK STR PPIMAGE_J1_RESID_B + JPEG_BIN1_RESID_SHUTTER STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_FLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_DOMEFLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_SKYFLAT STR PPIMAGE_J1_RESID_F + JPEG_BIN1_RESID_FRINGE STR PPIMAGE_J1_RESID_R + JPEG_BIN2_RESID_BIAS STR PPIMAGE_J2_RESID_B + JPEG_BIN2_RESID_DARK STR PPIMAGE_J2_RESID_B + JPEG_BIN2_RESID_SHUTTER STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_FLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_DOMEFLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_SKYFLAT STR PPIMAGE_J2_RESID_F + JPEG_BIN2_RESID_FRINGE STR PPIMAGE_J2_RESID_R + END + # Processing raw data + DEFAULT METADATA + CHIP STR PPIMAGE_OBDSFRA + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + END +END + +FITS METADATA +# BITPIX is the bits per pixel for writing the output data +# COMP = NONE|RICE|GZIP|HCOMPRESS|PLIO is the compression algorithm +# TILE.[XYZ] are the tile sizes. 0 means entire the dimension, so (0,1,1) forms tiles from rows +# NOISE [0..16] is the number of "noise bits" to preserve when quantising floating point data; 16 for no loss +# HSCALE is the scale factor for lossy compression with HCOMPRESS; 0 or 1 for none; 2*RMS --> 10x compression +# HSMOOTH is the smoothing to apply to HCOMPRESSed data when decompressing; 0 for none + +# BITPIX(S32) is the bits per pixel for writing the output data +# COMPRESSION(STR) = NONE|RICE|GZIP|HCOMPRESS|PLIO is the compression algorithm +# TILE.[XYZ](S32) are the tile sizes. 0 means entire the dimension, so (0,1,1) forms tiles from rows +# NOISE(S32) [0..16] is the number of "noise bits" to preserve when quantising floating point data +# HSCALE(S32) is the scale factor for lossy compression with HCOMPRESS; 0 or 1 for none; 2*RMS --> 10x compression +# HSMOOTH(S32) is the smoothing to apply to HCOMPRESSed data when decompressing; 0 for none +# SCALING(STR) = NONE|RANGE|STDEV_POSITIVE|STDEV_NEGATIVE|STDEV_BOTH|MANUAL is the scaling scheme +# BSCALE(F32) is the manual scaling to apply (when SCALING = MANUAL) +# BZERO(F32) is the manual zero-point to apply (when SCALING = MANUAL) +# STDEV.BITS(S32) is the number of bits to map to a standard deviation (when SCALING = STDEV_*) +# STDEV.NUM(F32) is the number of standard deviations to the edge (when SCALING = STDEV_NEGATIVE|STDEV_POSITIVE) +# FLOAT(STR) is the name of a custom floating-point type + + DET_IMAGE METADATA + BITPIX S32 -32 + END + DET_MASK METADATA + BITPIX S32 8 + END + DET_WEIGHT METADATA + BITPIX S32 -32 + END + + SKY_IMAGE METADATA + BITPIX S32 -32 + END + SKY_MASK METADATA + BITPIX S32 8 + END + SKY_WEIGHT METADATA + BITPIX S32 -32 + END + + COMPRESSED_POSITIVE METADATA + BITPIX S32 16 + SCALING STR STDEV_POSITIVE + STDEV.BITS S32 4 + STDEV.NUM F32 10 + COMPRESSION STR RICE + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + COMPRESSED_MASK METADATA + COMPRESSION STR PLIO + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + COMPRESSED_SUBTRACTION METADATA + BITPIX S32 16 + SCALING STR STDEV_BOTH + STDEV.BITS S32 4 + STDEV.NUM F32 5 + COMPRESSION STR RICE + TILE.X S32 0 + TILE.Y S32 1 + TILE.Z S32 1 + NOISE S32 8 + END + +END + +FILERULES METADATA + ### Redirections + PSASTRO.INPUT STR PSASTRO.INPUT.CMF + PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.SPL + PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF + PSPHOT.OUTPUT STR PSPHOT.OUT.CMF.SPL + + ### input file definitions + ### use @DETDB entries to get the detrend images from the database + ### replace @DETDB with @FILES if you want to require it from the + ### command line, or with an explicit name to require a specific file + TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE + + ## files used by ppImage + PPIMAGE.INPUT INPUT @FILES CHIP IMAGE + PPIMAGE.MASK INPUT @DETDB CELL IMAGE + PPIMAGE.BIAS INPUT @DETDB CELL IMAGE + PPIMAGE.DARK INPUT @DETDB CELL DARK + PPIMAGE.FLAT INPUT @DETDB CELL IMAGE + PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE + PPIMAGE.SHUTTER INPUT @DETDB CELL IMAGE + + ## Files used by ppMerge + PPMERGE.INPUT INPUT @FILES CHIP IMAGE + PPMERGE.INPUT.MASK INPUT @FILES CHIP MASK + PPMERGE.INPUT.WEIGHT INPUT @FILES CHIP WEIGHT + + ## files used to build and apply the flat-field correction images + DVOCORR.INPUT INPUT @FILES CHIP IMAGE + DVOCORR.REFHEAD INPUT @FILES CHIP HEADER + DVOFLAT.INPUT INPUT @FILES CHIP IMAGE + DVOFLAT.CORR INPUT @DETDB CHIP IMAGE + + ## files used by psphot + PSPHOT.LOAD INPUT @FILES CHIP IMAGE + PSPHOT.INPUT INPUT @FILES CHIP IMAGE + PSPHOT.MASK INPUT @FILES CHIP MASK + PSPHOT.WEIGHT INPUT @FILES CHIP WEIGHT + PSPHOT.PSF.LOAD INPUT @FILES CHIP PSF + PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF + + ## files used by psastro + PSASTRO.INPUT.CMF INPUT @FILES CHIP CMF + + ## files used by pswarp + PSWARP.INPUT INPUT @FILES CHIP IMAGE + PSWARP.SKYCELL INPUT @FILES FPA IMAGE + PSWARP.WEIGHT INPUT @FILES CHIP WEIGHT + PSWARP.MASK INPUT @FILES CHIP MASK + PSWARP.ASTROM INPUT @FILES CHIP CMF + + PPSUB.INPUT INPUT none.fits FPA IMAGE + PPSUB.INPUT.MASK INPUT none.fits FPA MASK + PPSUB.INPUT.WEIGHT INPUT none.fits FPA WEIGHT + PPSUB.REF INPUT none.fits FPA IMAGE + PPSUB.REF.MASK INPUT none.fits FPA MASK + PPSUB.REF.WEIGHT INPUT none.fits FPA WEIGHT + + PPSTACK.INPUT INPUT none.fits FPA IMAGE + PPSTACK.INPUT.MASK INPUT none.fits FPA MASK + PPSTACK.INPUT.WEIGHT INPUT none.fits FPA WEIGHT + + PPSTAMP.INPUT INPUT @FILES CHIP IMAGE + + PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE + PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + + + ### output file definitions + TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT + PPIMAGE.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT + PPIMAGE.OUTPUT.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.mask.fits MASK NONE CHIP TRUE SPLIT + PPIMAGE.OUTPUT.WEIGHT OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits WEIGHT NONE CHIP TRUE SPLIT + PPIMAGE.CHIP OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits IMAGE NONE CHIP TRUE SPLIT + PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.ch.mask.fits MASK NONE CHIP TRUE SPLIT + PPIMAGE.CHIP.WEIGHT OUTPUT {OUTPUT}.{CHIP.NAME}.ch.wt.fits WEIGHT NONE CHIP TRUE SPLIT + PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.fpa1.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.OUTPUT.FPA2 OUTPUT {OUTPUT}.fpa2.fits IMAGE NONE FPA TRUE NONE + PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + + PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE + PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE + PPIMAGE.BIN1 OUTPUT {OUTPUT}.{CHIP.NAME}.b1.fits IMAGE NONE CHIP TRUE SPLIT + PPIMAGE.BIN2 OUTPUT {OUTPUT}.{CHIP.NAME}.b2.fits IMAGE NONE CHIP TRUE SPLIT + + PPMERGE.OUTPUT.MASK OUTPUT {OUTPUT}.{CHIP.NAME}.fits MASK NONE CHIP TRUE SPLIT + PPMERGE.OUTPUT.BIAS OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT + PPMERGE.OUTPUT.DARK OUTPUT {OUTPUT}.{CHIP.NAME}.fits DARK NONE CHIP TRUE SPLIT + PPMERGE.OUTPUT.SHUTTER OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT + PPMERGE.OUTPUT.FLAT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT + PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.{CHIP.NAME}.fits FRINGE NONE CHIP TRUE SPLIT + PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.sigma.fits IMAGE NONE CHIP TRUE NONE + PPMERGE.OUTPUT.COUNT OUTPUT {OUTPUT}.count.fits IMAGE NONE CHIP TRUE NONE + + DVOCORR.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fc.fits IMAGE NONE CHIP TRUE NONE + DVOFLAT.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.co.fits IMAGE NONE CHIP TRUE NONE + + PSPHOT.RESID OUTPUT {OUTPUT}.{CHIP.NAME}.res.fits IMAGE NONE CHIP TRUE NONE + PSPHOT.BACKGND OUTPUT {OUTPUT}.{CHIP.NAME}.bck.fits IMAGE NONE CHIP TRUE NONE + PSPHOT.BACKSUB OUTPUT {OUTPUT}.{CHIP.NAME}.sub.fits IMAGE NONE CHIP TRUE NONE + PSPHOT.BACKMDL OUTPUT {OUTPUT}.{CHIP.NAME}.mdl.fits IMAGE NONE CHIP TRUE NONE + PSPHOT.BACKMDL.STDEV OUTPUT {OUTPUT}.{CHIP.NAME}.mdd.fits IMAGE NONE CHIP TRUE NONE + + PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT}.{CHIP.NAME} RAW NONE CHIP TRUE NONE + PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.{CHIP.NAME}.sx SX NONE CHIP TRUE NONE + PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.{CHIP.NAME}.obj OBJ NONE CHIP TRUE NONE + PSPHOT.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.cmf CMF NONE CHIP TRUE NONE + PSPHOT.OUT.CMF.MEF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE + + PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.{CHIP.NAME}.psf PSF NONE CHIP TRUE NONE + + SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.{CHIP.NAME}.mnt.png KAPA NONE CHIP TRUE NONE + SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.{CHIP.NAME}.psf.png KAPA NONE CHIP TRUE NONE + SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.{CHIP.NAME}.dap.png KAPA NONE CHIP TRUE NONE + + PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE SPLIT + PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE SPLIT + PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE TOGETHER + + PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE + PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE + PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + + SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + + PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE + PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + + PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE + PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.weight.fits WEIGHT NONE FPA TRUE NONE + + PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE + PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + + PPSIM.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT + PPSIM.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE + + PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE + PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE + + LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE + LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE +END + +# FPA file defines properties of a possible input|output object +# user can set the filename (I|O), filename rules (O), or abstract source (@FILES, @DETDB) (I) +# user can set the extension name, if used +# user can set the file type (IMAGE, JPEG, RAW, SX, OBJ, CMP, CMF) : but these are not variable in most cases! +# user can set the file depth: only valid for output files +# user can set the data depth: must be >= file depth +# user can set the file format: only valid for newly created FPAs +# user can set the colormap, scaling method, scaling range (JPEG only) +# user can set the extension name for the data and header segments (CMF only) + + +EXTNAME.RULES METADATA + CMF.HEAD STR {CHIP.NAME}.hdr + CMF.DATA STR {CHIP.NAME}.psf + PSF.HEAD STR {CHIP.NAME}.hdr + PSF.TABLE STR {CHIP.NAME}.psf_model + PSF.RESID STR {CHIP.NAME}.psf_resid +END + +BLANK.HEADERS METADATA + FPA.TIME STR MJD-OBS + FPA.EXPOSURE STR EXPTIME + FPA.AIRMASS STR AIRMASS +END Index: /tags/pap_merge_030828/ippconfig/simtest/filerules.mdc =================================================================== --- /tags/pap_merge_030828/ippconfig/simtest/filerules.mdc (revision 17232) +++ /tags/pap_merge_030828/ippconfig/simtest/filerules.mdc (revision 17232) @@ -0,0 +1,137 @@ +### File rules for PHU=FPA, EXT=NONE + + +PSASTRO.INPUT STR PSASTRO.INPUT.CMF +PSASTRO.OUTPUT STR PSASTRO.OUTPUT.CMF +PSASTRO.OUTPUT.MEF STR PSASTRO.OUTPUT.CMF +PSPHOT.OUTPUT STR PSPHOT.OUTPUT.CMF + +### input file definitions +TYPE INPUT FILENAME.RULE DATA.LEVEL FILE.TYPE +PPIMAGE.INPUT INPUT none.fits FPA IMAGE +INPUT.MASK INPUT none.fits FPA MASK +INPUT.WEIGHT INPUT none.fits FPA WEIGHT +INPUT.PSF INPUT none.fits READOUT PSF +INPUT.SRC INPUT none.fits READOUT CMF +PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE +PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE +PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE +PPIMAGE.MASK INPUT @DETDB CHIP IMAGE + +## Files used by ppMerge +PPMERGE.INPUT INPUT @FILES CHIP IMAGE +PPMERGE.INPUT.MASK INPUT @FILES CHIP MASK +PPMERGE.INPUT.WEIGHT INPUT @FILES CHIP WEIGHT + +## files used by psphot +PSPHOT.LOAD INPUT @FILES CHIP IMAGE +PSPHOT.INPUT INPUT none.fits CHIP IMAGE + +PSWARP.INPUT INPUT none.fits FPA IMAGE +PSWARP.WEIGHT INPUT none.fits FPA WEIGHT +PSWARP.MASK INPUT none.fits FPA MASK +PSWARP.SKYCELL INPUT none.fits FPA IMAGE +PSWARP.ASTROM INPUT none.fits FPA CMF + +PSASTRO.INPUT.CMP INPUT none.fits CHIP CMP +PSASTRO.INPUT.CMF INPUT none.fits CHIP CMF +PSPHOT.PSF.LOAD INPUT none.psf CHIP PSF +PSPHOT.INPUT.CMF INPUT none.fits CHIP CMF + +PPSUB.INPUT INPUT none.fits FPA IMAGE +PPSUB.INPUT.MASK INPUT none.fits FPA MASK +PPSUB.INPUT.WEIGHT INPUT none.fits FPA WEIGHT +PPSUB.REF INPUT none.fits FPA IMAGE +PPSUB.REF.MASK INPUT none.fits FPA MASK +PPSUB.REF.WEIGHT INPUT none.fits FPA WEIGHT + +PPSTACK.INPUT INPUT none.fits FPA IMAGE +PPSTACK.INPUT.MASK INPUT none.fits FPA MASK +PPSTACK.INPUT.WEIGHT INPUT none.fits FPA WEIGHT + +PPSTAMP.INPUT INPUT none.fits FPA IMAGE + +PPARITH.INPUT.IMAGE INPUT @FILES CHIP IMAGE +PPARITH.INPUT.MASK INPUT @FILES CHIP MASK + +### output file definitions +TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT +PPIMAGE.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.OUTPUT.DETMASK OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE +PPIMAGE.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE +PPIMAGE.CHIP OUTPUT {OUTPUT}.ch.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.CHIP.MASK OUTPUT {OUTPUT}.ch.mask.fits MASK NONE FPA TRUE NONE +PPIMAGE.CHIP.WEIGHT OUTPUT {OUTPUT}.ch.wt.fits WEIGHT NONE FPA TRUE NONE +PPIMAGE.OUTPUT.FPA1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.OUTPUT.FPA2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE + +PPIMAGE.JPEG1 OUTPUT {OUTPUT}.b1.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE +PPIMAGE.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE +PPIMAGE.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + +PPMERGE.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE +PPMERGE.OUTPUT.BIAS OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.DARK OUTPUT {OUTPUT}.fits DARK NONE CHIP TRUE NONE +PPMERGE.OUTPUT.SHUTTER OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.FLAT OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.fits FRINGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.sigma.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.COUNT OUTPUT {OUTPUT}.count.fits IMAGE NONE CHIP TRUE NONE + +DVOCORR.OUTPUT OUTPUT {OUTPUT}.fc.fits IMAGE NONE FPA TRUE NONE +DVOFLAT.OUTPUT OUTPUT {OUTPUT}.co.fits IMAGE NONE FPA TRUE NONE + +PSPHOT.RESID OUTPUT {OUTPUT}.res.fits IMAGE NONE FPA TRUE NONE +PSPHOT.BACKGND OUTPUT {OUTPUT}.bck.fits IMAGE NONE FPA TRUE NONE +PSPHOT.BACKSUB OUTPUT {OUTPUT}.sub.fits IMAGE NONE FPA TRUE NONE +PSPHOT.BACKMDL OUTPUT {OUTPUT}.mdl.fits IMAGE NONE FPA TRUE NONE + +PSPHOT.OUTPUT.RAW OUTPUT {OUTPUT} RAW NONE FPA TRUE NONE +PSPHOT.OUTPUT.SX OUTPUT {OUTPUT}.sx SX NONE FPA TRUE NONE +PSPHOT.OUTPUT.OBJ OUTPUT {OUTPUT}.obj OBJ NONE FPA TRUE NONE +PSPHOT.OUTPUT.CMP OUTPUT {OUTPUT}.cmp CMP NONE FPA TRUE NONE +PSPHOT.OUTPUT.CMF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE +PSPHOT.PSF.SAVE OUTPUT {OUTPUT}.psf PSF NONE FPA TRUE NONE + +SOURCE.PLOT.MOMENTS OUTPUT {OUTPUT}.mnt.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.PSFMODEL OUTPUT {OUTPUT}.psf.png KAPA NONE FPA TRUE NONE +SOURCE.PLOT.APRESID OUTPUT {OUTPUT}.dap.png KAPA NONE FPA TRUE NONE + +PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.smp CMP NONE FPA TRUE NONE +PSASTRO.OUTPUT.CMF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE NONE + +PSWARP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PSWARP.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE +PSWARP.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE +PSWARP.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE +PSWARP.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE + +SKYCELL.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE +SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE + +PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE +PPSUB.OUTPUT.WEIGHT OUTPUT {OUTPUT}.wt.fits WEIGHT NONE FPA TRUE NONE + +PPSTACK.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTACK.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK NONE FPA TRUE NONE +PPSTACK.OUTPUT.WEIGHT OUTPUT {OUTPUT}.weight.fits WEIGHT NONE FPA TRUE NONE + +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.CHIP OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF + +PPSIM.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE SIMTEST +PPSIM.SOURCES OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE + +PPARITH.OUTPUT.IMAGE OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPARITH.OUTPUT.MASK OUTPUT {OUTPUT}.fits MASK NONE CHIP TRUE NONE + +LOG.IMFILE OUTPUT {OUTPUT}.imfile.log TEXT NONE FPA TRUE NONE +LOG.EXP OUTPUT {OUTPUT}.exp.log TEXT NONE FPA TRUE NONE + +TRACE.IMFILE OUTPUT {OUTPUT}.imfile.trace TEXT NONE FPA TRUE NONE +TRACE.EXP OUTPUT {OUTPUT}.exp.trace TEXT NONE FPA TRUE NONE Index: /tags/pap_merge_030828/ppMerge/.cvsignore =================================================================== --- /tags/pap_merge_030828/ppMerge/.cvsignore (revision 17232) +++ /tags/pap_merge_030828/ppMerge/.cvsignore (revision 17232) @@ -0,0 +1,12 @@ +Makefile +Makefile.in +aclocal.m4 +autom4te.cache +compile +config.log +config.status +configure +depcomp +install-sh +missing +test Index: /tags/pap_merge_030828/ppMerge/Makefile.am =================================================================== --- /tags/pap_merge_030828/ppMerge/Makefile.am (revision 17232) +++ /tags/pap_merge_030828/ppMerge/Makefile.am (revision 17232) @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = *~ core core.* Index: /tags/pap_merge_030828/ppMerge/autogen.sh =================================================================== --- /tags/pap_merge_030828/ppMerge/autogen.sh (revision 17232) +++ /tags/pap_merge_030828/ppMerge/autogen.sh (revision 17232) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ppMerge +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +#($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $LIBTOOlIZE installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" +# DIE=1 +#} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOHEADER installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +#$LIBTOOLIZE --copy --force || echo "$LIBTOOlIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /tags/pap_merge_030828/ppMerge/configure.ac =================================================================== --- /tags/pap_merge_030828/ppMerge/configure.ac (revision 17232) +++ /tags/pap_merge_030828/ppMerge/configure.ac (revision 17232) @@ -0,0 +1,30 @@ +AC_PREREQ(2.61) + +AC_INIT([ppMerge], [1.1.0], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([src]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDCFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC_C99 +AC_PROG_INSTALL +dnl AC_PROG_LIBTOOL +AC_SYS_LARGEFILE + +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0]) +PKG_CHECK_MODULES([PSMODULE], [psmodules >= 1.0.0]) +PKG_CHECK_MODULES([PPSTATS], [ppStats >= 1.0.0]) + +IPP_STDOPTS +CFLAGS="${CFLAGS=} -Wall -Werror" + +AC_CONFIG_FILES([ + Makefile + src/Makefile +]) +AC_OUTPUT Index: /tags/pap_merge_030828/ppMerge/src/.cvsignore =================================================================== --- /tags/pap_merge_030828/ppMerge/src/.cvsignore (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/.cvsignore (revision 17232) @@ -0,0 +1,7 @@ +.deps +Makefile +Makefile.in +ppMerge +config.h +config.h.in +stamp-h1 Index: /tags/pap_merge_030828/ppMerge/src/Makefile.am =================================================================== --- /tags/pap_merge_030828/ppMerge/src/Makefile.am (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/Makefile.am (revision 17232) @@ -0,0 +1,25 @@ +bin_PROGRAMS = ppMerge + +ppMerge_CFLAGS = $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) +ppMerge_LDFLAGS = $(PPSTATS_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS) + +ppMerge_SOURCES = \ + ppMerge.c \ + ppMergeArguments.c \ + ppMergeCamera.c \ + ppMergeFiles.c \ + ppMergeScaleZero.c \ + ppMergeLoop.c \ + ppMergeMask.c + + +noinst_HEADERS = \ + ppMerge.h + +CLEANFILES = *~ + +clean-local: + -rm -f TAGS +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` Index: /tags/pap_merge_030828/ppMerge/src/ppMerge.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMerge.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMerge.c (revision 17232) @@ -0,0 +1,155 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppMerge.h" + +//#include "ppMem.h" + +// Yet to do: +// +// 1. Mask pixels with less than the minimum number of electrons +// 2. Sampling factor for background measurement +// 3. On/off pairs + +int main(int argc, char **argv) +{ + psLibInit(NULL); + psMemSetThreadSafety(false); + psTimerStart(TIMERNAME); + + psExit exitValue = PS_EXIT_SUCCESS; // Exit value for program + + pmConfig *config = pmConfigRead(&argc, argv, PPMERGE_RECIPE); // Configuration + if (!config) { + psErrorStackPrint(stderr, "Error reading configuration."); + exitValue = PS_EXIT_CONFIG_ERROR; + goto die; + } + + if (!ppMergeArguments(argc, argv, config)) { + psErrorStackPrint(stderr, "Error reading arguments."); + exitValue = PS_EXIT_CONFIG_ERROR; + goto die; + } + + ppMergeType type = psMetadataLookupS32(NULL, config->arguments, "TYPE"); // Type of frame + switch (type) { + case PPMERGE_TYPE_MASK: + if (!ppMergeMask(config)) { + psErrorStackPrint(stderr, "Error generating mask."); + exitValue = PS_EXIT_DATA_ERROR; + goto die; + } + break; + case PPMERGE_TYPE_BIAS: + case PPMERGE_TYPE_DARK: + case PPMERGE_TYPE_SHUTTER: + case PPMERGE_TYPE_FLAT: + case PPMERGE_TYPE_FRINGE: + if (!ppMergeScaleZero(config)) { + psErrorStackPrint(stderr, "Error getting scale and zero-points."); + exitValue = PS_EXIT_DATA_ERROR; + goto die; + } + if (!ppMergeLoop(config)) { + psErrorStackPrint(stderr, "Error performing merge."); + exitValue = PS_EXIT_PROG_ERROR; + goto die; + } + break; + default: + psAbort("Invalid frame type: %x", type); + } + + + // Output the statistics + bool mdok; // Status of MD lookup + psString statsName = psMetadataLookupStr(&mdok, config->arguments, "STATS.NAME"); // Statistics file name + if (mdok && statsName && strlen(statsName) > 0) { + psString resolved = pmConfigConvertFilename(statsName, config, true); // Resolved filename + FILE *statsFile = fopen(resolved, "w"); // Output statistics file + if (!statsFile) { + psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.", resolved); + psFree(resolved); + exitValue = PS_EXIT_CONFIG_ERROR; + goto die; + } + psFree(resolved); + psMetadata *stats = psMetadataLookupMetadata(&mdok, config->arguments, "STATS.DATA"); // Statistics + if (!stats) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find statistics"); + exitValue = PS_EXIT_PROG_ERROR; + goto die; + } + psString statsOut = psMetadataConfigFormat(stats); // String to write out + fprintf(statsFile, "%s", statsOut); + psFree(statsOut); + fclose(statsFile); + } + + + + +#if 0 + // Set various tasks (define optional operations) + ppMergeOptions *options = ppMergeOptionsParse(config); + if (!options) { + psErrorStackPrint(stderr, "Unable to parse options."); + exit(EXIT_FAILURE); + } + + // Check the inputs + ppMergeData *data = ppMergeCheckInputs(options, config); + if (!data) { + psErrorStackPrint(stderr, "Not enough valid input files."); + exit(EXIT_FAILURE); + } + + if (options->mask) { + // Generate a mask + ppMergeMask(data, options, config); + } else { + + psImage *scale = NULL; // The scalings + psImage *zero = NULL; // The zeros + psArray *shutters = NULL; // The shutter correction data + + // Measure the background in each image + ppMergeScaleZero(&scale, &zero, &shutters, data, options, config); + + // Do the combination and write + ppMergeCombine(scale, zero, shutters, data, options, config); + + psFree(scale); + psFree(zero); + psFree(shutters); + } + + // Output the statistics + if (data->statsFile && data->stats) { + psString statsOut = psMetadataConfigFormat(data->stats); // String to write out + fprintf(data->statsFile, "%s", statsOut); + psFree(statsOut); + } + + // Cleaning up + psFree(data); + psFree(options); +#endif + + die: + psTrace("ppSub", 1, "Finished at %f sec\n", psTimerMark("ppSub")); + psTimerStop(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exitValue); +} Index: /tags/pap_merge_030828/ppMerge/src/ppMerge.h =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMerge.h (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMerge.h (revision 17232) @@ -0,0 +1,82 @@ +#ifndef PP_MERGE_H +#define PP_MERGE_H + +#define TIMERNAME "ppMerge" // Name for timer +#define PPMERGE_RECIPE "PPMERGE" // Recipe name + +// Type of frame to merge +typedef enum { + PPMERGE_TYPE_UNKNOWN, // Unknown type + PPMERGE_TYPE_BIAS, // Bias frame + PPMERGE_TYPE_DARK, // (Multi-)Dark frame + PPMERGE_TYPE_MASK, // Mask frame + PPMERGE_TYPE_SHUTTER, // Shutter frame + PPMERGE_TYPE_FLAT, // Flat-field frame (dome or sky) + PPMERGE_TYPE_FRINGE, // Fringe frame +} ppMergeType; + +// Files, for activation +typedef enum { + PPMERGE_FILES_ALL, // All files + PPMERGE_FILES_INPUT, // Input files + PPMERGE_FILES_OUTPUT // Output files +} ppMergeFiles; + +// Parse command-line arguments and recipe +bool ppMergeArguments(int argc, char *argv[], // Command-line arguments + pmConfig *config // Configuration + ); + +// Set up camera files +bool ppMergeCamera(pmConfig *config // Configuration + ); + +// Measure scale and zero-points +bool ppMergeScaleZero(pmConfig *config // Configuration + ); + +// Main loop to do the merging +bool ppMergeLoop(pmConfig *config // Configuration + ); + +// Main loop for masks +bool ppMergeMask(pmConfig *config // Configuration + ); + +// Read nominated input file +bool ppMergeFileReadInput(const pmConfig *config, // Configuration + pmReadout *readout, // Readout into which to read + int num, // Number of file in sequence + int rows // Number of rows to read at once + ); + +// Open nominated input file +bool ppMergeFileOpenInput(pmConfig *config, // Configuration + const pmFPAview *view, // View to open + int num // Number of file in sequence + ); + +// Set the data level for files specified by name; return an array of the files +psArray *ppMergeFileDataLevel(const pmConfig *config, // Configuration + const char *name, // Name of files + pmFPALevel level // Level for file data level + ); + +// Activate/deactivate a list of files +bool ppMergeFileActivate(const pmConfig *config, // Configuration + ppMergeFiles files, // Files to turn on/off + bool state // Activation state + ); + +// Activate/deactivate a single element for a list; return array of files +psArray *ppMergeFileActivateSingle(const pmConfig *config, // Configuration + ppMergeFiles files, // Files to turn on/off + bool state, // Activation state + int num // Number of file in sequence + ); + +// Return name of output pmFPAfile +psString ppMergeOutputFile(const pmConfig *config // Configuration + ); + +#endif Index: /tags/pap_merge_030828/ppMerge/src/ppMergeArguments.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeArguments.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeArguments.c (revision 17232) @@ -0,0 +1,343 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "ppMerge.h" + +// Print usage information and die +static void usage(const char *program, // Name of the program + psMetadata *arguments // Command-line arguments + ) +{ + fprintf(stderr, "\nPan-STARRS Detrend Merging\n\n"); + fprintf(stderr, "Usage: %s INPUT.mdc OUTPUT_ROOT\n" + "where INPUTS.mdc contains various METADATAs, each with:\n" + "\tIMAGE(STR): Image filename\n" + "\tMASK(STR): Mask filename\n" + "\tWEIGHT(STR) Weight map filename\n" + "where MASK and WEIGHT are optional.", + program); + fprintf(stderr, "\n"); + psArgumentHelp(arguments); +} + +// Get a float-point value from the command-line or recipe, and add it to the arguments +#define VALUE_ARG_RECIPE_FLOAT(ARGNAME, RECIPENAME, TYPE) { \ + ps##TYPE value = psMetadataLookup##TYPE(NULL, arguments, ARGNAME); \ + if (isnan(value)) { \ + bool mdok; \ + value = psMetadataLookup##TYPE(&mdok, recipe, RECIPENAME); \ + if (!mdok) { \ + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find %s in recipe %s", \ + RECIPENAME, PPMERGE_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMetadataAdd##TYPE(config->arguments, PS_LIST_TAIL, RECIPENAME, 0, NULL, value); \ +} + +// Get an integer value from the command-line or recipe, and add it to the arguments +#define VALUE_ARG_RECIPE_INT(ARGNAME, RECIPENAME, TYPE, UNSET) { \ + ps##TYPE value = psMetadataLookup##TYPE(NULL, arguments, ARGNAME); \ + if (value == UNSET) { \ + bool mdok; \ + value = psMetadataLookup##TYPE(&mdok, recipe, RECIPENAME); \ + if (!mdok) { \ + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find %s in recipe %s", \ + RECIPENAME, PPMERGE_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMetadataAdd##TYPE(config->arguments, PS_LIST_TAIL, RECIPENAME, 0, NULL, value); \ +} + +// Get a boolean from the command-line or recipe, and add it to the arguments if either is set +#define VALUE_ARG_RECIPE_BOOL(ARGNAME, RECIPENAME) { \ + bool value = (psMetadataLookupBool(NULL, arguments, ARGNAME) || \ + psMetadataLookupBool(NULL, recipe, RECIPENAME)); \ + psMetadataAddBool(config->arguments, PS_LIST_TAIL, RECIPENAME, 0, NULL, value); \ +} + +// Get a statistic name from the command-line or recipe, and add the enum to the arguments +#define VALUE_ARG_RECIPE_STAT(ARGNAME, RECIPENAME) { \ + const char *stat = psMetadataLookupStr(NULL, arguments, ARGNAME); \ + if (!stat) { \ + stat = psMetadataLookupStr(NULL, recipe, RECIPENAME); \ + if (!stat) { \ + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to find %s in recipe %s", \ + RECIPENAME, PPMERGE_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMetadataAddS32(config->arguments, PS_LIST_TAIL, RECIPENAME, 0, NULL, psStatsOptionFromString(stat)); \ +} + +// Get a string from the command-line or recipe, and add to the arguments +#define VALUE_ARG_RECIPE_STR(ARGNAME, RECIPENAME) { \ + const char *str = psMetadataLookupStr(NULL, arguments, ARGNAME); \ + if (!str) { \ + str = psMetadataLookupStr(NULL, recipe, RECIPENAME); \ + if (!str) { \ + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to find %s in recipe %s", \ + RECIPENAME, PPMERGE_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMetadataAddStr(config->arguments, PS_LIST_TAIL, RECIPENAME, 0, NULL, str); \ +} + +// Get a string from the command-line or recipe, and add the appropriate mask value to the arguments +#define VALUE_ARG_RECIPE_MASK(ARGNAME, RECIPENAME) { \ + const char *str = psMetadataLookupStr(NULL, arguments, ARGNAME); \ + if (!str) { \ + str = psMetadataLookupStr(NULL, recipe, RECIPENAME); \ + if (!str) { \ + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to find %s in recipe %s", \ + RECIPENAME, PPMERGE_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMaskType mask = pmConfigMask(str, config); \ + psMetadataAddU8(config->arguments, PS_LIST_TAIL, RECIPENAME, 0, NULL, mask); \ +} + +// Get a string value from the command-line and add it to the target +static bool valueArgStr(psMetadata *arguments, // Command-line arguments + const char *argName, // Argument name in the command-line arguments + const char *mdName, // Name for value in the metadata + psMetadata *target // Target metadata to which to add value + ) +{ + psString value = psMetadataLookupStr(NULL, arguments, argName); // Value of interest + if (value && strlen(value) > 0) { + return psMetadataAddStr(target, PS_LIST_TAIL, mdName, 0, NULL, value); + } + return false; +} + +bool ppMergeArguments(int argc, char *argv[], pmConfig *config) +{ + assert(config); + + psMetadata *arguments = psMetadataAlloc(); // Command-line arguments + psMetadataAddStr(arguments, PS_LIST_TAIL, "-type", 0, "Type of calibration frame", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-stats", 0, "MDC file to hold statistics ", NULL); + // Standard combination parameters + psMetadataAddS32(arguments, PS_LIST_TAIL, "-rows", 0, "Rows to read per scan", 0); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-sample", 0, "Sampling factor for background", 0); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-iter", 0, "Number of rejection iterations", 0); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-rej", 0, "Rejection threshold (sigma)", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-fraclow", 0, "Fraction of low pixels to discard", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-frachigh", 0, "Fraction of low pixels to discard", NAN); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-nkeep", 0, "Minimum number of pixels in stack to keep", 0); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-weights", 0, "Use image weights in combination?", false); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-maskval", 0, "Mask value for input data", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-combine", 0, "Statistic to use for combination", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-mean", 0, "Statistic to use to measure the mean", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-stdev", 0, "Statistic to use to measure the stdev", NULL); + + // Fringe construction parameters + psMetadataAddS32(arguments, PS_LIST_TAIL, "-fringe-num", 0, "Number of fringe regions", 0); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-fringe-size", 0, "Half-size of fringe regions", 0); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-fringe-xsmooth", 0, "Number of smoothing regions in x", 0); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-fringe-ysmooth", 0, "Number of smoothing regions in y", 0); + + // Shutter construction parameters + psMetadataAddS32(arguments, PS_LIST_TAIL, "-shutter-size", 0, "Size for shutter measurement regions", 0); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-shutter-iter", 0, "Number of iterations for shutter", 0); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-shutter-rej", 0, "Rejection limit for shutter", NAN); + + // Mask construction parameters + psMetadataAddF32(arguments, PS_LIST_TAIL, "-mask-suspect", 0, "Threshold for suspect pixels (sigma)", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-mask-bad", 0, "Threshold for bad pixels (sigma)", NAN); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask-mode", 0, "Mode to identify bad pixels", NULL); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-mask-grow", 0, "Number of pixels to grow final mask", 0); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask-growval", 0, "Value to give grown mask pixels", NULL); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-mask-chip", 0, "Measure mask statistics by chip?", false); + + if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 3) { + usage(argv[0], arguments); + goto ERROR; + } + + unsigned int numBad = 0; // Number of bad lines + psMetadata *inputs = psMetadataConfigRead(NULL, &numBad, argv[1], false); // Information about inputs + if (!inputs || numBad > 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to cleanly read MDC file with inputs."); + goto ERROR; + } + psMetadataAddMetadata(config->arguments, PS_LIST_TAIL, "INPUTS", 0, + "Metadata with input details", inputs); + psFree(inputs); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, + "Root name of the output image list", argv[2]); + + valueArgStr(arguments, "-stats", "STATS.NAME", config->arguments); + + + // Set the type of calibration frame + const char *typeStr = psMetadataLookupStr(NULL, arguments, "-type"); // Type of calibration + if (strlen(typeStr) <= 0) { + psError(PS_ERR_UNKNOWN, false, "No -type specified."); + goto ERROR; + } + ppMergeType type = PPMERGE_TYPE_UNKNOWN; // Enumerated type for frame type + if (strcasecmp(typeStr, "BIAS") == 0) { + type = PPMERGE_TYPE_BIAS; + } else if (strcasecmp(typeStr, "DARK") == 0) { + type = PPMERGE_TYPE_DARK; + } else if (strcasecmp(typeStr, "FLAT") == 0 || strcasecmp(typeStr, "SKYFLAT") == 0 || + strcasecmp(typeStr, "DOMEFLAT") == 0) { + type = PPMERGE_TYPE_FLAT; + } else if (strcasecmp(typeStr, "FRINGE") == 0) { + type = PPMERGE_TYPE_FRINGE; + } else if (strcasecmp(typeStr, "SHUTTER") == 0) { + type = PPMERGE_TYPE_SHUTTER; + } else if (strcasecmp(typeStr, "MASK") == 0 || + strcasecmp(typeStr, "DARKMASK") == 0 || + strcasecmp(typeStr, "FLATMASK") == 0) { + type = PPMERGE_TYPE_MASK; + } else { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unrecognised image type: %s", typeStr); + goto ERROR; + } + psMetadataAddS32(config->arguments, PS_LIST_TAIL, "TYPE", 0, "Type of calibration frame", type); + + // Need to set the camera before the recipes can be read + if (!ppMergeCamera(config)) { + psError(PS_ERR_UNKNOWN, false, "Unable to setup cameras."); + goto ERROR; + } + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPMERGE_RECIPE); // Recipe for ppSim + if (!recipe) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find recipe %s", PPMERGE_RECIPE); + goto ERROR; + } + + // Standard combination parameters + VALUE_ARG_RECIPE_INT("-rows", "ROWS", S32, 0); + VALUE_ARG_RECIPE_INT("-sample", "SAMPLE", S32, 0); + VALUE_ARG_RECIPE_INT("-iter", "ITER", S32, 0); + VALUE_ARG_RECIPE_FLOAT("-rej", "REJ", F32); + VALUE_ARG_RECIPE_FLOAT("-fraclow", "FRACLOW", F32); + VALUE_ARG_RECIPE_FLOAT("-frachigh", "FRACHIGH", F32); + VALUE_ARG_RECIPE_INT("-nkeep", "NKEEP", S32, 0); + VALUE_ARG_RECIPE_BOOL("-weights", "WEIGHTS"); + VALUE_ARG_RECIPE_MASK("-maskval", "MASKVAL"); + VALUE_ARG_RECIPE_STAT("-combine", "COMBINE"); + VALUE_ARG_RECIPE_STAT("-mean", "MEAN"); + VALUE_ARG_RECIPE_STAT("-stdev", "STDEV"); + + // Fringe construction parameters + VALUE_ARG_RECIPE_INT("-fringe-num", "FRINGE.NUM", S32, 0); + VALUE_ARG_RECIPE_INT("-fringe-size", "FRINGE.SIZE", S32, 0); + VALUE_ARG_RECIPE_INT("-fringe-xsmooth", "FRINGE.XSMOOTH", S32, 0); + VALUE_ARG_RECIPE_INT("-fringe-ysmooth", "FRINGE.YSMOOTH", S32, 0); + + // Shutter construction parameters + VALUE_ARG_RECIPE_INT("-shutter-size", "SHUTTER.SIZE", S32, 0); + + // Mask construction parameters + VALUE_ARG_RECIPE_FLOAT("-mask-suspect", "MASK.SUSPECT", F32); + VALUE_ARG_RECIPE_FLOAT("-mask-bad", "MASK.BAD", F32); + VALUE_ARG_RECIPE_INT("-mask-grow", "MASK.GROW", S32, 0); + VALUE_ARG_RECIPE_MASK("-mask-growval", "MASK.GROWVAL"); + VALUE_ARG_RECIPE_BOOL("-mask-chip", "MASK.CHIPSTATS"); + + + const char *maskModeStr = psMetadataLookupStr(NULL, arguments, "-mask-mode"); // Mode to identify bad pix + if (!maskModeStr) { + maskModeStr = psMetadataLookupStr(NULL, recipe, "MASK.MODE"); + if (!maskModeStr) { + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "No mask mode specified in recipe."); + goto ERROR; + } + } + pmMaskIdentifyMode maskMode = pmMaskIdentifyModeFromString(maskModeStr); + if (maskMode == PM_MASK_ID_NONE) { + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Invalid mask mode %s", maskModeStr); + goto ERROR; + } + psMetadataAddS32(config->arguments, PS_LIST_TAIL, "MASK.MODE", 0, "Mode for mask identification", + maskMode); + + + if (type == PPMERGE_TYPE_DARK) { + psMetadata *ordinates = psMetadataLookupMetadata(NULL, recipe, "DARK.ORDINATES"); // Ordinates info + psArray *translated = psArrayAllocEmpty(psListLength(ordinates->list)); // Translated version + + psMetadataIterator *iter = psMetadataIteratorAlloc(ordinates, PS_LIST_HEAD, NULL); // Iterator + psMetadataItem *item; // Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + int order = 0; // Polynomial order + bool scale = false; // Scale values? + float min = NAN, max = NAN; // Minimum and maximum values for scaling + switch (item->type) { + case PS_TYPE_S32: + order = item->data.S32; + break; + case PS_DATA_METADATA: + order = psMetadataLookupS32(NULL, item->data.md, "ORDER"); + bool mdok; // Status of MD lookup + scale = psMetadataLookupBool(&mdok, item->data.md, "SCALE"); + min = psMetadataLookupF32(&mdok, item->data.md, "MIN"); + max = psMetadataLookupF32(&mdok, item->data.md, "MAX"); + break; + default: + psError(PS_ERR_BAD_PARAMETER_TYPE, true, + "Type of DARK.ORDINATES entry %s (%x) is not METADATA or S32", + item->name, item->type); + return false; + } + if (order <= 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "ORDER not positive (%d) for DARK.ORDINATES %s", + order, item->name); + return false; + } + + pmDarkOrdinate *ord = pmDarkOrdinateAlloc(item->name, order); + ord->scale = scale; + ord->min = min; + ord->max = max; + psArrayAdd(translated, translated->n, ord); + psFree(ord); + } + psFree(iter); + + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "DARK.ORDINATES", 0, + "Ordinates to fit for dark", translated); + psFree(translated); // Drop reference + + psString darkNorm = psMetadataLookupStr(NULL, recipe, "DARK.NORM"); // Normalisation concept + if (darkNorm && strcmp(darkNorm, "NONE") != 0) { + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "DARK.NORM", 0, + "Normalisation concept for dark", darkNorm); + } + } + + +#if 0 + // Add concepts for scale and zero + // XXX These have never been used + psMetadataItem *scaleItem = psMetadataItemAllocF32("PPMERGE.SCALE", "Scaling for ppMerge", NAN); + psMetadataItem *zeroItem = psMetadataItemAllocF32("PPMERGE.ZERO", "Zero offset for ppMerge", NAN); + pmConceptRegister(scaleItem, NULL, NULL, false, PM_FPA_LEVEL_CELL); + pmConceptRegister(zeroItem, NULL, NULL, false, PM_FPA_LEVEL_CELL); + psFree(scaleItem); + psFree(zeroItem); +#endif + + return true; + +ERROR: + psFree(arguments); + return false; +} + Index: /tags/pap_merge_030828/ppMerge/src/ppMergeCamera.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeCamera.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeCamera.c (revision 17232) @@ -0,0 +1,322 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "ppMerge.h" + +// Define an output file, with its own FPA +bool outputFile(pmConfig *config, // Configuration + const char *name, // Name of output file + pmFPAfileType type, // Type of file + const char *description, // Description of file + psMetadata *format, // Camera format + pmFPAview *view // View for PHU + ) +{ + assert(config); + assert(name && strlen(name) > 0); + assert(view); + + // Output image + pmFPA *fpa = pmFPAConstruct(config->camera); // FPA to contain the output + if (!fpa) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to construct an FPA from camera configuration."); + return false; + } + + pmFPAfile *output = pmFPAfileDefineOutput(config, fpa, name); + psFree(fpa); // Drop reference + if (!output) { + psError(PS_ERR_IO, false, "Unable to generate output file from %s", name); + return false; + } + if (output->type != type) { + psError(PS_ERR_IO, true, "%s is not of type %s", name, pmFPAfileStringFromType(type)); + return false; + } + output->save = true; + + if (!pmFPAAddSourceFromView(fpa, description, view, format)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate output FPA."); + return false; + } + + return true; +} + + +bool ppMergeCamera(pmConfig *config) +{ + bool haveMasks = false; // Do we have masks? + bool haveWeights = false; // Do we have weight maps? + + ppMergeType type = psMetadataLookupS32(NULL, config->arguments, "TYPE"); // Type of frame + psMetadata *inputs = psMetadataLookupMetadata(NULL, config->arguments, "INPUTS"); // The inputs info + psMetadataIterator *iter = psMetadataIteratorAlloc(inputs, PS_LIST_HEAD, NULL); // Iterator + psMetadataItem *item; // Item from iteration + int numFiles = 0; // Number of files + while ((item = psMetadataGetAndIncrement(iter))) { + if (item->type != PS_DATA_METADATA) { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, + "Component %s of the input metadata is not of type METADATA", item->name); + psFree(iter); + return false; + } + + psMetadata *input = item->data.md; // The input metadata of interest + + psString image = psMetadataLookupStr(NULL, input, "IMAGE"); // Name of image + if (!image || strlen(image) == 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Component %s lacks IMAGE of type STR", item->name); + psFree(iter); + return false; + } + + bool mdok; + psString mask = psMetadataLookupStr(&mdok, input, "MASK"); // Name of mask + psString weight = psMetadataLookupStr(&mdok, input, "WEIGHT"); // Name of weight map + + // Add the image file + psArray *imageFiles = psArrayAlloc(1); // Array of filenames for this FPA + imageFiles->data[0] = psMemIncrRefCounter(image); + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "IMAGE.FILENAMES", PS_META_REPLACE, + "Filenames of image files", imageFiles); + psFree(imageFiles); + + bool found = false; // Found the file? + pmFPAfile *imageFile = pmFPAfileDefineFromArgs(&found, config, "PPMERGE.INPUT", "IMAGE.FILENAMES"); + if (!imageFile || !found) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from image %d (%s)", numFiles, image); + return false; + } + if (imageFile->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPMERGE.INPUT is not of type IMAGE"); + return false; + } + + // Optionally add the mask file + if (mask && strlen(mask) > 0) { + psArray *maskFiles = psArrayAlloc(1); // Array of filenames for this FPA + maskFiles->data[0] = psMemIncrRefCounter(mask); + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "MASK.FILENAMES", PS_META_REPLACE, + "Filenames of mask files", maskFiles); + psFree(maskFiles); + + bool status; + pmFPAfile *maskFile = pmFPAfileBindFromArgs(&status, imageFile, config, "PPMERGE.INPUT.MASK", + "MASK.FILENAMES"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from mask %d (%s)", numFiles, mask); + return false; + } + if (maskFile->type != PM_FPA_FILE_MASK) { + psError(PS_ERR_IO, true, "PPMERGE.INPUT.MASK is not of type MASK"); + return false; + } + haveMasks = true; + } + + // Optionally add the weight file + if (weight && strlen(weight) > 0) { + haveWeights = true; + psArray *weightFiles = psArrayAlloc(1); // Array of filenames for this FPA + weightFiles->data[0] = psMemIncrRefCounter(weight); + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "WEIGHT.FILENAMES", PS_META_REPLACE, + "Filenames of weight files", weightFiles); + psFree(weightFiles); + + bool status; + pmFPAfile *weightFile = pmFPAfileBindFromArgs(&status, imageFile, config, "PPMERGE.INPUT.WEIGHT", + "WEIGHT.FILENAMES"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from weight %d (%s)", numFiles, weight); + return false; + } + if (weightFile->type != PM_FPA_FILE_WEIGHT) { + psError(PS_ERR_IO, true, "PPMERGE.INPUT.WEIGHT is not of type WEIGHT"); + return false; + } + haveWeights = true; + } + + numFiles++; + } + psFree(iter); + psMetadataRemoveKey(config->arguments, "IMAGE.FILENAMES"); + if (psMetadataLookup(config->arguments, "MASK.FILENAMES")) { + psMetadataRemoveKey(config->arguments, "MASK.FILENAMES"); + } + if (psMetadataLookup(config->arguments, "WEIGHT.FILENAMES")) { + psMetadataRemoveKey(config->arguments, "WEIGHT.FILENAMES"); + } + if (psMetadataLookup(config->arguments, "PSF.FILENAMES")) { + psMetadataRemoveKey(config->arguments, "PSF.FILENAMES"); + } + + psMetadataAddS32(config->arguments, PS_LIST_TAIL, "INPUTS.NUM", 0, "Number of input files", numFiles); + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "INPUTS.MASKS", 0, "Got input masks?", haveMasks); + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "INPUTS.WEIGHTS", 0, + "Got input weights?", haveWeights); + + + // Check that all the inputs are consistent + +// Check an FPA level to ensure the camera format and PHU are consistent across all input files +#define CHECK_LEVEL(HDU, CHIP, CELL) { \ + if (HDU) { \ + if (!phuView) { \ + phuView = pmFPAviewAlloc(0); \ + phuView->chip = CHIP; \ + phuView->cell = CELL; \ + } else if ((phuView->chip != (CHIP)) && (phuView->cell != (CELL))) { \ + psError(PS_ERR_UNKNOWN, true, "Differing PHU for input %d", i); \ + psFree(phuView); \ + return false; \ + } \ + if (!format) { \ + format = (HDU)->format; \ + } else if (format != (HDU)->format) { \ + psError(PS_ERR_UNKNOWN, true, "Camera format %d doesn't match: %p vs %p", \ + i, format, (HDU)->format); \ + psFree(phuView); \ + return false; \ + } \ + continue; \ + } \ +} + + psMetadata *format = NULL; // Camera format + pmFPAview *phuView = NULL; // View to PHU + for (int i = 0; i < numFiles; i++) { + pmFPAfile *input = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT", i); // File of interest + pmFPA *fpa = input->fpa; // FPA of interest + CHECK_LEVEL(fpa->hdu, -1, -1); + psArray *chips = fpa->chips; // Array of chips + for (int j = 0; j < chips->n; j++) { + pmChip *chip = chips->data[j]; // Chip of interest + CHECK_LEVEL(chip->hdu, j, -1); + psArray *cells = chip->cells; // Array of cells + for (int k = 0; k < cells->n; k++) { + pmCell *cell = cells->data[k]; // Cell of interest + CHECK_LEVEL(cell->hdu, j, k); + } + } + } + if (!phuView || !format) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find PHU for input files."); + psFree(phuView); + return false; + } + + // Cull chips and cells that don't have data + // Otherwise the abundance of metadata in the concepts (esp. for GPC) can overload the memory + for (int i = 0; i < numFiles; i++) { + pmFPAfile *input = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT", i); // File of interest + pmFPA *fpa = input->fpa; // FPA of interest + psArray *chips = fpa->chips; // Array of chips in output + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + psArray *cells = chip->cells; // Array of cells + int culled = 0; // Number of culled cells + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; + pmHDU *hdu = pmHDUGetLowest(fpa, chip, cell); // HDU for cell + if (!hdu || hdu->blankPHU) { + psFree(cell->concepts); + cell->concepts = NULL; + cell->data_exists = false; + cell->file_exists = false; + culled++; + } + } + if (culled == cells->n) { + psFree(chip->concepts); + chip->concepts = NULL; + chip->data_exists = false; + chip->file_exists = false; + } + } + } + + // Count the cells + { + int numCells = 0; // Number of cells + pmFPAfile *input = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT", 0); // Representative file + pmFPA *fpa = input->fpa; // FPA for file + psArray *chips = fpa->chips; // Array of chips + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + psArray *cells = chip->cells; // Array of cells + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; // Cell of interest + pmHDU *hdu = pmHDUGetLowest(fpa, chip, cell); // HDU that would have data + if (hdu && !hdu->blankPHU) { + numCells++; + } + } + } + + psMetadataAddS32(config->arguments, PS_LIST_TAIL, "INPUTS.CELLS", 0, "Number of cells in input", + numCells); + } + + // Output image + pmFPA *fpa = pmFPAConstruct(config->camera); // FPA to contain the output + if (!fpa) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to construct an FPA from camera configuration."); + psFree(phuView); + return false; + } + + psString outName = ppMergeOutputFile(config); // Name of output file + + pmFPAfileType fileType = PM_FPA_FILE_NONE; // Type of output file + switch (type) { + case PPMERGE_TYPE_BIAS: + fileType = PM_FPA_FILE_IMAGE; + break; + case PPMERGE_TYPE_DARK: + fileType = PM_FPA_FILE_DARK; + break; + case PPMERGE_TYPE_MASK: + fileType = PM_FPA_FILE_MASK; + break; + case PPMERGE_TYPE_SHUTTER: + fileType = PM_FPA_FILE_IMAGE; + break; + case PPMERGE_TYPE_FLAT: + fileType = PM_FPA_FILE_IMAGE; + break; + case PPMERGE_TYPE_FRINGE: + fileType = PM_FPA_FILE_FRINGE; + default: + psAbort("Unknown frame type: %x", type); + } + + if (!outputFile(config, outName, fileType, "Merged detrend", format, phuView)) { + psFree(outName); + psFree(phuView); + return false; + } + psFree(outName); + + if (!outputFile(config, "PPMERGE.OUTPUT.SIGMA", PM_FPA_FILE_IMAGE, "Merge sigma", format, phuView)) { + psFree(phuView); + return false; + } + + if (!outputFile(config, "PPMERGE.OUTPUT.COUNT", PM_FPA_FILE_IMAGE, "Merged count", format, phuView)) { + psFree(phuView); + return false; + } + + psFree(phuView); + + return true; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeCheckInputs.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeCheckInputs.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeCheckInputs.c (revision 17232) @@ -0,0 +1,176 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeCheckInputs.h" +#include "ppMergeData.h" + +// Check input files to make sure everything's consistent +ppMergeData *ppMergeCheckInputs(ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + ppMergeData *data = ppMergeDataAlloc(); // The data, to return + + // Output file + psString outName = psMetadataLookupStr(NULL, config->arguments, "OUTPUT"); // The output file name + assert(outName); // It should be there! + outName = pmConfigConvertFilename(outName, config, true); + data->outFile = psFitsOpen(outName, "w"); // Output FITS file + if (!data->outFile) { + // There's no point in continuing if we can't open the output + psErrorStackPrint(stderr, "Can't open output image: %s\n", outName); + psFree(outName); + exit(EXIT_FAILURE); + } + psFree(outName); + + // Statistics file + psString statsName = psMetadataLookupStr(NULL, config->arguments, "-stats"); // Name for statistics file + if (statsName && strlen(statsName) > 0) { + psString resolved = pmConfigConvertFilename(statsName, config, true); // Resolved filename + data->statsFile = fopen(resolved, "w"); + if (!data->statsFile) { + psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved); + psFree(resolved); + psFree(data); + return NULL; + } + psFree(resolved); + } + + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); + if (!data->in) { + data->in = psArrayAlloc(filenames->n); + } + if (!data->files) { + data->files = psArrayAlloc(filenames->n); + } + int numGood = 0; // Number of good files + for (int i = 0; i < filenames->n; i++) { + psString name = filenames->data[i]; // The name of the file + if (!name || strlen(name) == 0) { + continue; + } + psTrace("ppMerge", 1, "Checking input file %s....\n", name); + psString resolved = pmConfigConvertFilename(name, config, false); // Resolved file name + psFits *inFile = psFitsOpen(resolved, "r"); // The FITS file to read + if (!inFile) { + psLogMsg(__func__, PS_LOG_WARN, "Unable to open input file %s --- ignored.\n", resolved); + // Kick it out + psFree(filenames->data[i]); + filenames->data[i] = NULL; + continue; + } + psFree(resolved); + psMetadata *header = psFitsReadHeader(NULL, inFile); // The FITS (primary) header + data->files->data[i] = inFile; + + // The formats must be identical. The chief reason for this is so that we know what output format to + // use. I guess one could specify a different output format on the command line, but how do we + // generate a PHU for that? Perhaps we could revisit this restriction in the future (construct an + // FPAview from the specified camera format configuration, and use pmFPAAddSourceFromView), but for + // now it's less hassle just to limit the output format to be the input format. + if (!options->format) { + options->format = pmConfigCameraFormatFromHeader(config, header, true); + if (!options->format) { + psLogMsg(__func__, PS_LOG_WARN, "Unable to identify camera format for input file %s --- " + "ignored.\n", name); + // Kick it out + psFree(header); + data->in->data[i] = NULL; + continue; + } + } else { + bool valid = false; + if (!pmConfigValidateCameraFormat(&valid, options->format, header)) { + psError (PS_ERR_UNKNOWN, false, "Error in config scripts\n"); + exit (PS_EXIT_CONFIG_ERROR); + } + if (!valid) { + psLogMsg(__func__, PS_LOG_WARN, "Input file %s doesn't match camera format --- ignored.\n", name); + // Kick it out + psFree(header); + data->in->data[i] = NULL; + continue; + } + } + + pmFPA *fpa = pmFPAConstruct(config->camera); + pmFPAview *view = pmFPAAddSourceFromHeader(fpa, header, options->format); + psFree(view); + + // Cull chips and cells that don't have data + // Otherwise the abundance of metadata in the concepts (esp. for GPC) can overload the memory + psArray *chips = fpa->chips; // Array of chips in output + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + psArray *cells = chip->cells; // Array of cells + int culled = 0; // Number of culled cells + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; + pmHDU *hdu = pmHDUGetLowest(fpa, chip, cell); // HDU for cell + if (!hdu || hdu->blankPHU) { + psFree(cell->concepts); + cell->concepts = NULL; + culled++; + } + } + if (culled == cells->n) { + psFree(chip->concepts); + chip->concepts = NULL; + } + } + data->in->data[i] = fpa; + + + // Use the first valid input as the basis for the output --- including the header + if (!data->out) { + psTrace("ppMerge", 5, "Constructing output using %s as a template.\n", name); + data->out = pmFPAConstruct(config->camera); + pmFPAview *view = pmFPAAddSourceFromHeader(data->out, header, options->format); + psFree(view); + } + psFree(header); + + psTrace("ppMerge", 3, "%s checks out.\n", name); + numGood++; + } + + // Count the cells + int numCells = 0; // Number of cells in the output FPA + psArray *chips = data->out->chips; // Array of chips in output + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + if (!chip) { + continue; + } + psArray *cells = chip->cells; // Array of cells + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; + if (cell) { + numCells++; + } + } + } + data->numCells = numCells; + psTrace("ppMerge", 3, "Output has %d cells.\n", numCells); + + psTrace("ppMerge", 3, "We have %d good inputs.\n", numGood); + if (numGood > 1) { + return data; + } + + psFree(data); + return NULL; +} + + Index: /tags/pap_merge_030828/ppMerge/src/ppMergeCheckInputs.h =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeCheckInputs.h (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeCheckInputs.h (revision 17232) @@ -0,0 +1,13 @@ +#ifndef PP_MERGE_CHECK_INPUTS_H +#define PP_MERGE_CHECK_INPUTS_H + +#include +#include "ppMergeOptions.h" +#include "ppMergeData.h" + +// Check input files to make sure everything's consistent +ppMergeData *ppMergeCheckInputs(ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +#endif Index: /tags/pap_merge_030828/ppMerge/src/ppMergeCombine.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeCombine.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeCombine.c (revision 17232) @@ -0,0 +1,434 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeCombine.h" +#include "ppMergeVersion.h" + +#define TESTING + + +#if 0 +static FILE *dumpFile = NULL; + +static psMemId mbAlloc(psMemBlock *mb) +{ + if (!dumpFile) { + dumpFile = fopen("memBlocks.dat", "w"); + } + fprintf(dumpFile, "Alloc: %12lu\t%12zd\t%s:%d\n", mb->id, mb->userMemorySize, + mb->file, mb->lineno); + return 1; +} + +static void dumpDone(void) +{ + fclose(dumpFile); + exit(EXIT_FAILURE); +} +#endif + +#if 0 +static psMemId memId = 0; +static void memDump(void) +{ + psMemBlock **leaks = NULL; + int numLeaks = psMemCheckLeaks(memId, &leaks, NULL, true); + FILE *memFile = fopen("mem.dat", "w"); + fprintf(memFile, "# MemBlock Size Source\n"); + for (int i = 0; i < numLeaks; i++) { + psMemBlock *mb = leaks[i]; + fprintf(memFile, "%12lu\t%12zd\t%s:%d\n", mb->id, mb->userMemorySize, + mb->file, mb->lineno); + } + fclose(memFile); + psFree(leaks); +} +#endif + +#if 0 +static void memCheck(void) +{ + return; + if (psTraceGetLevel("ppMerge") > 9) { + psMemBlock **leaks = NULL; + int numLeaks = psMemCheckLeaks(0, &leaks, NULL, true); + size_t largestSize = 0; + psMemId largest = 0; + size_t totalSize = 0; + for (int i = 0; i < numLeaks; i++) { + psMemBlock *mb = leaks[i]; + totalSize += mb->userMemorySize; + if (mb->userMemorySize > largestSize) { + largestSize = mb->userMemorySize; + largest = mb->id; + } + } + psFree(leaks); + psTrace("ppMerge", 0, "Memory in use: %zd\n", totalSize); + psTrace("ppMerge", 0, "Largest block: %ld\n", largest); + psTrace("ppMerge", 0, "sbrk(): %p\n", sbrk(0)); + } + return; +} +#endif + + +// Combine the inputs +bool ppMergeCombine(psImage *scales, // Scales for each cell of each integration, or NULL + psImage *zeros, // Zeros for each cell of each integration, or NULL + psArray *shutters, // Shutter correction data for each cell, or NULL + ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + // Sanity checks + assert(!options->scale || scales); + assert(!scales || (scales->type.type == PS_TYPE_F32 && + scales->numCols == data->numCells && + scales->numRows == filenames->n)); + assert(!options->zero || zeros); + assert(!zeros || (zeros->type.type == PS_TYPE_F32 && + zeros->numCols == data->numCells && + zeros->numRows == filenames->n)); + assert(!options->shutter || shutters); + assert(!shutters || (shutters->n == data->numCells)); + + // Iterate over the FPA + pmFPA *fpa = data->out; // Output FPA + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + int cellNum = -1; // Cell number in the whole FPA + if (data->out->hdu) { + pmFPAUpdateNames(data->out, NULL, NULL); + } + pmFPAWrite(data->out, data->outFile, config->database, true, false); // Write header only + pmChip *chip; // Chip of interest + psRandom *rng = NULL; // Random number generator; required for building a mask + pmHDU *lastHDU = NULL; // Last HDU to be updated + if (options->mask) { + rng = psRandomAlloc(PS_RANDOM_TAUS, 0); + } + while ((chip = pmFPAviewNextChip(view, fpa, 1))) { + if (chip->hdu) { + // Data will exist soon + pmFPAUpdateNames(data->out, chip, NULL); + chip->data_exists = true; + } + pmChipWrite(chip, data->outFile, config->database, true, false); // Write header only + pmCell *cell; // Cell of interest + while ((cell = pmFPAviewNextCell(view, fpa, 1))) { + cellNum++; + + pmHDU *hdu = pmHDUGetLowest(data->out, chip, cell); // HDU for cell + if (!hdu || hdu->blankPHU) { + pmCellWrite(cell, data->outFile, config->database, true); // Write header only + continue; + } + if (cell->hdu) { + // Data will exist soon + pmFPAUpdateNames(data->out, chip, cell); + chip->data_exists = cell->data_exists = true; + } + pmReadout *readout = pmReadoutAlloc(cell); // Output readout of interest + psArray *stack = psArrayAlloc(filenames->n); // Stack of readouts to combine + psVector *cellScales = NULL; // Scales for this cell + if (scales) { + cellScales = psImageCol(NULL, scales, cellNum); + } + psVector *cellZeros = NULL; // Zeros for this cell + if (zeros) { + cellZeros = psImageCol(NULL, zeros, cellNum); + } + + // Read bit by bit + int numRead; // Number of inputs read + int numScan = 0; + + // Put version metadata into header + if (hdu && hdu != lastHDU) { + if (!hdu->header) { + hdu->header = psMetadataAlloc(); + } + ppMergeVersionMetadata(hdu->header); + lastHDU = hdu; + } + + float shutterRef = NAN; // Reference shutter correction + if (options->shutter) { + shutterRef = pmShutterCorrectionReference(shutters->data[cellNum]); + } + + do { + numRead = 0; + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + psFits *fits = data->files->data[i]; // FITS file handle + if (!fits) { + continue; + } + + if (!stack->data[i]) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + pmChip *chipIn = fpaIn->chips->data[view->chip]; // Input chip + pmCell *cellIn = chipIn->cells->data[view->cell]; // Input cell + stack->data[i] = pmReadoutAlloc(cellIn); // Input readout + } + + // Only reading and writing the first readout in each cell (plane 0) + bool readOK; + if (pmReadoutReadNext(&readOK, stack->data[i], fits, 0, options->rows)) { + if (!readOK) { + psError(PS_ERR_IO, false, "Failed to read concepts for cell.\n"); + psErrorStackPrint(stderr, "trouble reading data!\n"); + exit (1); + } + // If we're creating a bias or a dark, we don't want to generate a mask + if ((options->zero || options->scale || options->shutter || options->mask) && + options->combine->maskVal) { + pmReadoutSetMask(stack->data[i], options->satMask, options->badMask); + } + + // If we're combining with weights, we want to generate weights. + if (options->combine->weights && !options->dark) { + + // If it's a bias or dark, set the gain to zero: noise only contributed by read + if (!options->zero && !options->scale) { + pmReadoutSetWeight(stack->data[i], false); + } else { + pmReadoutSetWeight(stack->data[i], true); + } + } + + numRead++; + } else { + psTrace("ppMerge", 3, "Finished reading file %d, chip %d, cell %d, scan %d\n", + i, view->chip, view->cell, numScan); + } + + } + + psTrace("ppMerge", 5, "Chip %d, cell %d, scan %d\n", view->chip, view->cell, numScan); + if (numRead > 0) { + if (options->shutter) { + pmShutterCorrectionGenerate(readout, NULL, stack, shutterRef, shutters->data[cellNum], + options->shutterIter, options->shutterRej, + options->combine->maskVal); + } else if (options->dark) { + pmDarkCombine(cell, stack, options->darkOrdinates, options->darkNorm, + options->combine->iter, options->combine->rej, + options->combine->maskVal); + } else { + pmReadoutCombine(readout, stack, cellZeros, cellScales, options->combine); + } + } + numScan++; + + } while (numRead > 0); + + // Get list of cells for concepts averaging + psList *inCells = psListAlloc(NULL); // List of cells + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmCell *cellIn = pmFPAviewThisCell(view, data->in->data[i]); // Input cell + psListAdd(inCells, PS_LIST_TAIL, cellIn); + } + if (!pmConceptsAverageCells(cell, inCells, NULL, NULL, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average cell concepts."); + psFree(inCells); + return false; + } + psFree(inCells); + + psFree(stack); + +#if 0 + // Set the dark time for the output image, since we normalised + if (options->darktime) { + psMetadataItem *darkItem = psMetadataLookup(cell->concepts, "CELL.DARKTIME"); + darkItem->data.F32 = 1.0; + psMetadataItem *expItem = psMetadataLookup(cell->concepts, "CELL.EXPOSURE"); + expItem->data.F32 = 1.0; + } +#endif + + // Measure the fringes for this cell + // + // XXX Need to deal with multiple components: we will do this by building up the components + // Read the existing fringe measurements + // Use existing regions to measure fringe statistics + // Add the new fringe measurements to the existing fringe measurements. + // Write the appended fringe measurements. + // Read in the "output" file to get the existing components. + // Put the new readout into the cell after the existing readouts. + if (options->fringe && readout->image) { + pmFringeRegions *regions = pmFringeRegionsAlloc(options->fringeNum, options->fringeSize, + options->fringeSize, options->fringeSmoothX, + options->fringeSmoothY); // Fringe regions + pmFringeStats *fringe = pmFringeStatsMeasure(regions, readout, options->combine->maskVal); + psFree(regions); + if (!fringe) { + psError(PS_ERR_UNKNOWN, false, "Unable to measure fringe statistics.\n"); + psFree(readout); + return false; + } + + psArray *fringes = psArrayAlloc(1); // Array of fringes + fringes->data[0] = fringe; + + pmFringesFormat(cell, NULL, fringes); + psFree(fringes); // Drop reference + } + + if (readout->image) { + // Add MD5 information for cell + pmHDU *hdu = pmHDUFromCell(cell); // HDU that owns the cell + const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME"); + const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME"); + + psString headerName = NULL; // Header name for MD5 + psStringAppend(&headerName, "MD5_%s_%s", chipName, cellName); + + psVector *md5 = psImageMD5(readout->image); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, headerName, PS_META_REPLACE, + "Image MD5", md5string); + psFree(md5string); + psFree(headerName); + + + // Statistics on the merged cell + if (data->statsFile) { + if (!data->stats) { + data->stats = psMetadataAlloc(); + } + if (!ppStatsFPA(data->stats, data->out, view, + options->combine->maskVal | pmConfigMask("BLANK", config), + config)) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to generate stats for image.\n"); + return false; + } + } + } + + psFree(readout); // Drop reference + + + // We threw away the bias sections --- record this + psMetadataItem *biassecItem = psMetadataLookup(cell->concepts, "CELL.BIASSEC"); // Item of BIASSEC + psList *biassecList = biassecItem->data.V; // List of BIASSECs + while (psListRemove(biassecList, PS_LIST_TAIL)); // Removing all entries + + // Blow away the cell data + for (int i = 0; i < filenames->n; i++) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + if (!fpaIn) { continue; } // was not a valid input file + pmChip *chipIn = fpaIn->chips->data[view->chip]; // Input chip + pmCell *cellIn = chipIn->cells->data[view->cell]; // Input cell + pmCellFreeData(cellIn); + } + + // Write the pixels + if (cell->hdu && !cell->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out cell HDU.\n"); + if (options->dark) { + pmCellWriteDark(cell, data->outFile, config->database, false); + } else { + pmCellWrite(cell, data->outFile, config->database, false); + if (options->fringe) { + pmCellWriteTable(data->outFile, cell, "FRINGE"); + } + } + pmCellFreeData(cell); + } + } + + // Blow away the chip data + for (int i = 0; i < filenames->n; i++) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + if (!fpaIn) { continue; } // was not a valid input file + pmChip *chipIn = fpaIn->chips->data[view->chip]; // Input chip + pmChipFreeData(chipIn); + } + + // Write the pixels + if (chip->hdu && !chip->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out chip HDU.\n"); + if (options->dark) { + pmChipWriteDark(chip, data->outFile, config->database, false, false); + } else { + pmChipWrite(chip, data->outFile, config->database, false, false); + if (options->fringe) { + pmChipWriteTable(data->outFile, chip, "FRINGE"); + } + } + + pmChipFreeData(chip); + } + } + + // Blow away the FPA data + for (int i = 0; i < filenames->n; i++) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + if (!fpaIn) { continue; } // was not a valid input file + pmFPAFreeData(fpaIn); + } + + if (data->out->hdu && !data->out->hdu->blankPHU) { + // Write the pixels + psTrace("ppMerge", 5, "Writing out FPA HDU.\n"); + if (options->dark) { + pmFPAWriteDark(data->out, data->outFile, config->database, false, false); + } else { + pmFPAWrite(data->out, data->outFile, config->database, false, false); + if (options->fringe) { + pmFPAWriteTable(data->outFile, fpa, "FRINGE"); + } + } + } + + pmFPAFreeData(data->out); + + psFree(view); + psFree(rng); + + + // Get list of FPAs for concepts averaging + psList *inFPAs = psListAlloc(NULL); // List of FPAs + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmFPA *fpaIn = data->in->data[i]; // Input FPA + psListAdd(inFPAs, PS_LIST_TAIL, fpaIn); + } + + if (!pmConceptsAverageFPAs(fpa, inFPAs)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts."); + psFree(inFPAs); + return false; + } + psFree(inFPAs); + + + return true; + +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeCombine.h =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeCombine.h (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeCombine.h (revision 17232) @@ -0,0 +1,19 @@ +#ifndef PP_MERGE_COMBINE_H +#define PP_MERGE_COMBINE_H + +#include +#include + +#include "ppMergeData.h" +#include "ppMergeOptions.h" + +// Combine readouts +bool ppMergeCombine(psImage *scales, // Scales for each cell of each integration, or NULL + psImage *zeros, // Zeros for each cell of each integration, or NULL + psArray *shutters, // Shutter correction data for each cell, or NULL + ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +#endif Index: /tags/pap_merge_030828/ppMerge/src/ppMergeConfig.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeConfig.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeConfig.c (revision 17232) @@ -0,0 +1,93 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeConfig.h" + +#define DONT_USE_DB + +// Output usage information +static void usage(const char *programName, // Name of the program + psMetadata *arguments // Arguments list + ) +{ + printf("Merge multiple calibration frames into a master frame by stacking.\n\n" + "Usage:\n" + "\t%s OUTPUT.fits INPUT1.fits INPUT2.fits ...\n" + "\n", programName); + psArgumentHelp(arguments); + psFree(arguments); + exit(EXIT_FAILURE); +} + +pmConfig *ppMergeConfig(int argc, char **argv) +{ + pmConfig *config = pmConfigRead(&argc, argv, PPMERGE_RECIPE); + // Load the site-wide configuration information + if (! config) { + psErrorStackPrint(stderr, "Can't find site configuration!\n"); + exit(EXIT_FAILURE); + } + + psMetadata *arguments = psMetadataAlloc(); // Command-line arguments + psMetadataAddStr(arguments, PS_LIST_TAIL, "-type", 0, "Type of calibration frame", ""); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-zero", 0, "Subtract background?", false); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-scale", 0, "Scale by background?", false); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-exptime", 0, "Scale by the exposure time?", false); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-onoff", 0, "Number of on/off pairs", 0); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-stats", 0, "MDC file to hold statistics ", NULL); + + if (argc == 1) { + psFree(config); + usage(argv[0], arguments); + } + + // Parse the arguments + if (!psArgumentParse(arguments, &argc, argv) || argc < 3) { + psErrorStackPrint(stderr, "Unable to parse arguments."); + psFree(config); + usage(argv[0], arguments); + } + + // Add the output image to the arguments list + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", + argv[1]); + + psMetadataCopy(config->arguments, arguments); + psFree(arguments); + + // Everything remaining must be input files + if (argc - 2 <= 1) { + psErrorStackPrint(stderr, "No files to combine.\n"); + exit(EXIT_FAILURE); + } + psArray *files = psArrayAlloc(argc - 2); + for (int i = 2; i < argc; i++) { + files->data[i - 2] = psStringCopy(argv[i]); + } + psMetadataAddPtr(config->arguments, PS_LIST_TAIL, "INPUT", PS_DATA_ARRAY, + "Array of inputs images", files); + psFree(files); // Drop reference + +#ifndef DONT_USE_DB +#ifdef HAVE_PSDB + // Define database handle, if required + config->database = pmConfigDB(config); +#endif +#endif + + // Add concepts for scale and zero + psMetadataItem *scaleItem = psMetadataItemAllocF32("PPMERGE.SCALE", "Scaling for ppMerge", NAN); + psMetadataItem *zeroItem = psMetadataItemAllocF32("PPMERGE.ZERO", "Zero offset for ppMerge", NAN); + pmConceptRegister(scaleItem, NULL, NULL, false, PM_FPA_LEVEL_CELL); + pmConceptRegister(zeroItem, NULL, NULL, false, PM_FPA_LEVEL_CELL); + psFree(scaleItem); + psFree(zeroItem); + + return config; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeConfig.h =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeConfig.h (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeConfig.h (revision 17232) @@ -0,0 +1,10 @@ +#ifndef PP_MERGE_CONFIG_H +#define PP_MERGE_CONFIG_H + +#include + +// Get the configuration information +pmConfig *ppMergeConfig(int argc, char **argv // The standard command-line parameters (but pointer to number) + ); + +#endif Index: /tags/pap_merge_030828/ppMerge/src/ppMergeData.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeData.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeData.c (revision 17232) @@ -0,0 +1,50 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" + +// Free function for ppMergeData +static void mergeDataFree(ppMergeData *data // Data to free + ) +{ + if (data->files) { + for (long i = 0; i < data->files->n; i++) { + psFitsClose(data->files->data[i]); + data->files->data[i] = NULL; + } + psFree(data->files); + } + psFree(data->in); + psFree(data->out); + if (data->outFile) { + psFitsClose(data->outFile); + data->outFile = NULL; + } + if (data->statsFile) { + fclose(data->statsFile); + data->statsFile = NULL; + } + psFree(data->stats); +} + +// Allocator for ppMergeData +ppMergeData *ppMergeDataAlloc(void) +{ + ppMergeData *data = psAlloc(sizeof(ppMergeData)); // The data, to return + psMemSetDeallocator(data, (psFreeFunc)mergeDataFree); + + data->numCells = 0; + data->files = NULL; + data->in = NULL; + data->out = NULL; + data->outFile = NULL; + data->stats = NULL; + data->statsFile = NULL; + + return data; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeData.h =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeData.h (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeData.h (revision 17232) @@ -0,0 +1,23 @@ +#ifndef PP_MERGE_DATA_H +#define PP_MERGE_DATA_H + +#include +#include + +// Container for the data +typedef struct { + int numCells; // Number of (valid) cells in the FPA + psArray *files; // Input file pointers + psArray *in; // Input FPA structures + pmFPA *out; // Output FPA structure + psFits *outFile; // FITS file handle for output + psMetadata *stats; // Statistics on the combined image + FILE *statsFile; // File stream for statistics output +} ppMergeData; + + +// Allocator +ppMergeData *ppMergeDataAlloc(void); + + +#endif Index: /tags/pap_merge_030828/ppMerge/src/ppMergeErrorCodes.c.in =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeErrorCodes.c.in (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeErrorCodes.c.in (revision 17232) @@ -0,0 +1,26 @@ +/* + * The line + { PPMERGE_ERR_$X{ErrorCode}, "$X{ErrorDescription}"}, + * (without the Xs) + * will be replaced by values from errorCodes.dat + */ +#include "pslib.h" +#include "ppMergeErrorCodes.h" + +void ppMergeErrorRegister(void) +{ + static psErrorDescription errors[] = { + { PPMERGE_ERR_BASE, "First value we use; lower values belong to psLib" }, + { PPMERGE_ERR_${ErrorCode}, "${ErrorDescription}"}, + }; + static int nerror = PPMERGE_ERR_NERROR - PPMERGE_ERR_BASE; // number of values in enum + + for (int i = 0; i < nerror; i++) { + psErrorDescription *tmp = psAlloc(sizeof(psErrorDescription)); + p_psMemSetPersistent(tmp, true); + *tmp = errors[i]; + psErrorRegister(tmp, 1); + psFree(tmp); /* it's on the internal list */ + } + nerror = 0; // don't register more than once +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeErrorCodes.dat =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeErrorCodes.dat (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeErrorCodes.dat (revision 17232) @@ -0,0 +1,10 @@ +# +# This file is used to generate ppMergeErrorClasses.h +# +BASE = 700 First value we use; lower values belong to psLib +# these errors correspond to standard exit conditions +ARGUMENTS Incorrect arguments +SYS System error +CONFIG Problem in configure files +PROG Programming error +DATA invalid data Index: /tags/pap_merge_030828/ppMerge/src/ppMergeErrorCodes.h.in =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeErrorCodes.h.in (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeErrorCodes.h.in (revision 17232) @@ -0,0 +1,18 @@ +#if !defined(PPMERGE_ERROR_CODES_H) +#define PPMERGE_ERROR_CODES_H +/* + * The line + * PPMERGE_ERR_$X{ErrorCode}, + * (without the X) + * + * will be replaced by values from errorCodes.dat + */ +typedef enum { + PPMERGE_ERR_BASE = 512, + PPMERGE_ERR_${ErrorCode}, + PPMERGE_ERR_NERROR +} ppMergeErrorCode; + +void ppMergeErrorRegister(void); + +#endif Index: /tags/pap_merge_030828/ppMerge/src/ppMergeFiles.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeFiles.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeFiles.c (revision 17232) @@ -0,0 +1,214 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppMerge.h" + +const char *allFiles[] = { "PPMERGE.INPUT", "PPMERGE.INPUT.MASK", "PPMERGE.INPUT.WEIGHT", + "PPMERGE.OUTPUT", "PPMERGE.OUTPUT.COUNT", "PPMERGE.OUTPUT.SIGMA", + NULL }; // All files +const char *inputFiles[] = { "PPMERGE.INPUT", "PPMERGE.INPUT.MASK", "PPMERGE.INPUT.WEIGHT", + NULL }; // Input files +const char *outputFiles[] = { "PPMERGE.OUTPUT", "PPMERGE.OUTPUT.COUNT", "PPMERGE.OUTPUT.SIGMA", + NULL }; // Output files + +// Select file list based on enum +static const char **selectFiles(ppMergeFiles files) +{ + switch (files) { + case PPMERGE_FILES_ALL: return allFiles; + case PPMERGE_FILES_INPUT: return inputFiles; + case PPMERGE_FILES_OUTPUT: return outputFiles; + default: + psAbort("Invalid file option"); + } + return NULL; +} + +bool ppMergeFileReadInput(const pmConfig *config, pmReadout *readout, int num, int rows) +{ + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT", num); + if (!pmReadoutReadChunk(readout, file->fits, 0, rows, 0)) { + psError(PS_ERR_UNKNOWN, false, "Unable to read readout."); + return false; + } + bool mdok; // Status of MD lookup + if (psMetadataLookupBool(&mdok, config->arguments, "INPUTS.MASKS")) { + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT.MASK", num); + if (!pmReadoutReadChunkMask(readout, file->fits, 0, rows, 0)) { + psError(PS_ERR_UNKNOWN, false, "Unable to read readout mask."); + return false; + } + } + if (psMetadataLookupBool(&mdok, config->arguments, "INPUTS.WEIGHTS")) { + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT.WEIGHT", num); + if (!pmReadoutReadChunkWeight(readout, file->fits, 0, rows, 0)) { + psError(PS_ERR_UNKNOWN, false, "Unable to read readout weight."); + return false; + } + } + return true; +} + +bool ppMergeFileOpenInput(pmConfig *config, const pmFPAview *view, int num) +{ + { + pmFPAfile *input = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT", num); + pmFPAview *fileView = pmFPAviewForLevel(input->fileLevel, view); + if (!pmFPAfileOpen(input, fileView, config)) { + psError(PS_ERR_UNKNOWN, false, "Unable to open image file %d", num); + psFree(fileView); + return false; + } + psFree(fileView); + } + bool mdok; // Status of MD lookup + if (psMetadataLookupBool(&mdok, config->arguments, "INPUTS.MASKS")) { + pmFPAfile *mask = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT.MASK", num); // Mask file + pmFPAview *fileView = pmFPAviewForLevel(mask->fileLevel, view); + if (!pmFPAfileOpen(mask, fileView, config)) { + psError(PS_ERR_UNKNOWN, false, "Unable to open mask file %d", num); + psFree(fileView); + return false; + } + psFree(fileView); + } + if (psMetadataLookupBool(&mdok, config->arguments, "INPUTS.WEIGHTS")) { + pmFPAfile *weight = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT.WEIGHT", num); // Weight file + pmFPAview *fileView = pmFPAviewForLevel(weight->fileLevel, view); + if (!pmFPAfileOpen(weight, fileView, config)) { + psError(PS_ERR_UNKNOWN, false, "Unable to open weight file %d", num); + psFree(fileView); + return false; + } + psFree(fileView); + } + return true; +} + +psArray *ppMergeFileDataLevel(const pmConfig *config, const char *name, pmFPALevel level) +{ + assert(config); + assert(name); + + int numFiles = psMetadataLookupS32(NULL, config->arguments, "INPUTS.NUM"); // Number of input files + assert(numFiles > 0); + psArray *files = psArrayAlloc(numFiles); // Files of interest + for (int i = 0; i < numFiles; i++) { + pmFPAfile *file = pmFPAfileSelectSingle(config->files, name, i); // Image file + file->dataLevel = level; + file->freeLevel = level; + files->data[i] = psMemIncrRefCounter(file); + } + return files; +} + +bool ppMergeFileActivate(const pmConfig *config, ppMergeFiles files, bool state) +{ + assert(config); + + bool mdok; // Status of MD lookup + bool haveMasks = psMetadataLookupBool(&mdok, config->arguments, "INPUTS.MASKS"); // Do we have masks? + bool haveWeights = psMetadataLookupBool(&mdok, config->arguments, "INPUTS.WEIGHTS"); // Got weights? + + const char **fileList = selectFiles(files); // Files to activate + for (int i = 0; fileList[i] != NULL; i++) { + if (!haveMasks && strcmp(fileList[i], "PPMERGE.INPUT.MASK") == 0) { + continue; + } + if (!haveWeights && strcmp(fileList[i], "PPMERGE.INPUT.WEIGHT") == 0) { + continue; + } + psString name = NULL; // Name of file + if (strcmp(fileList[i], "PPMERGE.OUTPUT") == 0) { + name = ppMergeOutputFile(config); + } else { + name = psStringCopy(fileList[i]); + } + + if (!pmFPAfileActivate(config->files, state, name)) { + psError(PS_ERR_UNKNOWN, false, "Unable to activate file %s", name); + psFree(name); + return false; + } + psFree(name); + } + + return true; +} + + + +psArray *ppMergeFileActivateSingle(const pmConfig *config, ppMergeFiles files, bool state, int num) +{ + assert(config); + + bool mdok; // Status of MD lookup + bool haveMasks = psMetadataLookupBool(&mdok, config->arguments, "INPUTS.MASKS"); // Do we have masks? + bool haveWeights = psMetadataLookupBool(&mdok, config->arguments, "INPUTS.WEIGHTS"); // Got weights? + + psList *list = psListAlloc(NULL); // List of files + const char **fileList = selectFiles(files); // Files to activate + for (int i = 0; fileList[i] != NULL; i++) { + if (!haveMasks && strcmp(fileList[i], "PPMERGE.INPUT.MASK") == 0) { + continue; + } + if (!haveWeights && strcmp(fileList[i], "PPMERGE.INPUT.WEIGHT") == 0) { + continue; + } + + psString name = NULL; // Name of file + if (strcmp(fileList[i], "PPMERGE.OUTPUT") == 0) { + name = ppMergeOutputFile(config); + } else { + name = psStringCopy(fileList[i]); + } + + pmFPAfile *file = pmFPAfileActivateSingle(config->files, state, name, num); // Activated file + psFree(name); + psListAdd(list, PS_LIST_TAIL, file); + } + + psArray *array = psListToArray(list); + psFree(list); + + return array; +} + + +psString ppMergeOutputFile(const pmConfig *config) +{ + ppMergeType type = psMetadataLookupS32(NULL, config->arguments, "TYPE"); // Type of frame + const char *outSuffix = NULL; // Suffix for output file + switch (type) { + case PPMERGE_TYPE_BIAS: + outSuffix = "BIAS"; + break; + case PPMERGE_TYPE_DARK: + outSuffix = "DARK"; + break; + case PPMERGE_TYPE_MASK: + outSuffix = "MASK"; + break; + case PPMERGE_TYPE_SHUTTER: + outSuffix = "SHUTTER"; + break; + case PPMERGE_TYPE_FLAT: + outSuffix = "FLAT"; + break; + case PPMERGE_TYPE_FRINGE: + outSuffix = "FRINGE"; + default: + psAbort("Unknown frame type: %x", type); + } + + psString outName = NULL; // Name of output file + psStringAppend(&outName, "PPMERGE.OUTPUT.%s", outSuffix); + + return outName; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeLoop.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeLoop.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeLoop.c (revision 17232) @@ -0,0 +1,346 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include +#include +#include + +#include "ppMerge.h" + +bool ppMergeLoop(pmConfig *config) +{ + assert(config); + + psMetadata *arguments = config->arguments; // Arguments + ppMergeType type = psMetadataLookupS32(NULL, config->arguments, "TYPE"); // Type of frame + int numFiles = psMetadataLookupS32(NULL, arguments, "INPUTS.NUM"); // Number of input files + bool mdok; // Status of MD lookup + bool haveMasks = psMetadataLookupBool(&mdok, arguments, "INPUTS.MASKS"); // Do we have masks? + bool haveWeights = psMetadataLookupBool(&mdok, arguments, "INPUTS.WEIGHTS"); // Do we have weights? + + psArray *inputs = ppMergeFileDataLevel(config, "PPMERGE.INPUT", PM_FPA_LEVEL_READOUT); // Input images + psArray *masks = NULL, *weights = NULL; // Input masks and weights + if (haveMasks) { + masks = ppMergeFileDataLevel(config, "PPMERGE.INPUT.MASK", PM_FPA_LEVEL_READOUT); + } + if (haveWeights) { + weights = ppMergeFileDataLevel(config, "PPMERGE.INPUT.WEIGHT", PM_FPA_LEVEL_READOUT); + } + + // General combination parameters + int rows = psMetadataLookupS32(NULL, arguments, "ROWS"); // Number of rows to read per chunk + int iter = psMetadataLookupS32(NULL, arguments, "ITER"); // Number of rejection iterations + float rej = psMetadataLookupF32(NULL, arguments, "REJ"); // Rejection level + float fraclow = psMetadataLookupF32(NULL, arguments, "FRACLOW"); // Reject fraction of low pixels + float frachigh = psMetadataLookupF32(NULL, arguments, "FRACHIGH"); // Reject fraction of hi pixels + int nKeep = psMetadataLookupS32(NULL, arguments, "NKEEP"); // Minimum number of values to keep + psMaskType maskVal = psMetadataLookupU8(NULL, arguments, "MASKVAL"); // Value to mask + psStatsOptions combineStat = psMetadataLookupS32(NULL, arguments, "COMBINE"); // Combination statistic + bool useWeights = psMetadataLookupBool(NULL, arguments, "WEIGHTS"); // Use weights? + + // Fringe parameters + int fringeNum = psMetadataLookupS32(NULL, arguments, "FRINGE.NUM"); // Number of fringe points + int fringeSize = psMetadataLookupS32(NULL, arguments, "FRINGE.SIZE"); // Size of fringe regions + int fringeSmoothX = psMetadataLookupS32(NULL, arguments, "FRINGE.XSMOOTH"); // Smoothing regions in x + int fringeSmoothY = psMetadataLookupS32(NULL, arguments, "FRINGE.YSMOOTH"); // Smoothing regions in y + + pmCombineParams *combination = pmCombineParamsAlloc(combineStat); // Combination parameters + combination->maskVal = maskVal; + combination->blank = pmConfigMask("BLANK", config); + combination->nKeep = nKeep; + combination->fracHigh = frachigh; + combination->fracLow = fraclow; + combination->iter = iter; + combination->rej = rej; + combination->weights = useWeights; + + // Retrieve data placed on analysis + psVector *scales = NULL, *zeros = NULL; // Scale and zeroes for combination + psArray *shutters = NULL; // Shutter correction data + switch (type) { + case PPMERGE_TYPE_FRINGE: + zeros = psMetadataLookupPtr(NULL, arguments, "ZEROS"); + if (!zeros) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find ZEROS"); + goto ERROR; + } + // Flow through + case PPMERGE_TYPE_FLAT: + scales = psMetadataLookupPtr(NULL, arguments, "SCALES"); + if (!scales) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find SCALES"); + goto ERROR; + } + break; + case PPMERGE_TYPE_SHUTTER: + shutters = psMetadataLookupPtr(NULL, arguments, "SHUTTER"); + if (!shutters) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find SHUTTER"); + goto ERROR; + } + break; + case PPMERGE_TYPE_MASK: + case PPMERGE_TYPE_BIAS: + case PPMERGE_TYPE_DARK: + break; + default: + psAbort("Should never get here."); + } + + // Dark parameters + psArray *darkOrdinates = psMetadataLookupPtr(NULL, arguments, "DARK.ORDINATES"); // Dark info + psString darkNorm = psMetadataLookupStr(&mdok, arguments, "DARK.NORM"); // Dark normalisation + + psMetadata *stats = NULL; // Statistics for output + if (psMetadataLookup(config->arguments, "STATS.NAME")) { + stats = psMetadataAlloc(); + psMetadataAddMetadata(config->arguments, PS_LIST_TAIL, "STATS.DATA", 0, "Statistics output", stats); + } + + if (!ppMergeFileActivate(config, PPMERGE_FILES_ALL, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to activate files."); + goto ERROR; + } + psString outName = ppMergeOutputFile(config); // Name of output file + pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, outName); // Output file + psFree(outName); + assert(output && output->fpa); + pmFPA *outFPA = output->fpa; // Output FPA + pmFPAview *view = pmFPAviewAlloc(0); // View to component of interest + int cellNum = 0; // Index of cell + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + goto ERROR; + } + pmChip *outChip; // Chip of interest + while ((outChip = pmFPAviewNextChip(view, outFPA, 1))) { + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + goto ERROR; + } + pmCell *outCell; // Cell of interest + while ((outCell = pmFPAviewNextCell(view, outFPA, 1))) { + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + goto ERROR; + } + + pmHDU *hdu = pmHDUGetLowest(outFPA, outChip, outCell); // HDU for cell + if (!hdu || hdu->blankPHU) { + // No data here + continue; + } + + pmReadout *outRO = pmReadoutAlloc(outCell); + + psArray *readouts = psArrayAlloc(numFiles); // Input readouts + for (int i = 0; i < numFiles; i++) { + // We need to do some of the opening ourselves + if (!ppMergeFileOpenInput(config, view, i)) { + psError(PS_ERR_UNKNOWN, false, "Unable to open file %d", i); + goto ERROR; + } + + pmFPAfile *input = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT", i); + pmCell *inCell = pmFPAviewThisCell(view, input->fpa); // Input cell + readouts->data[i] = pmReadoutAlloc(inCell); + } + + float shutterRef = NAN; // Reference shutter correction + if (type == PPMERGE_TYPE_SHUTTER) { + shutterRef = pmShutterCorrectionReference(shutters->data[cellNum]); + } + + // Read convolutions by chunks + bool more = true; // More to read? + for (int numChunk = 0; more; numChunk++) { + psTrace("ppStack", 2, "Initial stack of chunk %d....\n", numChunk); + for (int i = 0; i < numFiles; i++) { + pmReadout *inRO = readouts->data[i]; // Input readout + + // Read a chunk from a file + #define READ_CHUNK(NAME,TYPE) { \ + pmFPAfile *file = pmFPAfileSelectSingle(config->files, NAME, i); \ + if (!pmReadoutReadChunk##TYPE(inRO, file->fits, 0, rows, 0)) { \ + psError(PS_ERR_IO, false, "Unable to read chunk %d for file %s %d", \ + numChunk, NAME, i); \ + psFree(readouts); \ + psFree(outRO); \ + goto ERROR; \ + } \ + } + + READ_CHUNK("PPMERGE.INPUT", /* Blank */); + if (haveMasks) { + READ_CHUNK("PPMERGE.INPUT.MASK", Mask); + } + if (haveWeights) { + READ_CHUNK("PPMERGE.INPUT.WEIGHT", Weight); + } + } + + switch (type) { + case PPMERGE_TYPE_SHUTTER: + if (!pmShutterCorrectionGenerate(outRO, NULL, readouts, shutterRef, + shutters->data[cellNum], iter, rej, maskVal)) { + psFree(readouts); + psFree(outRO); + goto ERROR; + } + break; + case PPMERGE_TYPE_DARK: + if (!pmDarkCombine(outCell, readouts, darkOrdinates, darkNorm, iter, rej, maskVal)) { + psFree(readouts); + psFree(outRO); + goto ERROR; + } + break; + case PPMERGE_TYPE_BIAS: + case PPMERGE_TYPE_FLAT: + case PPMERGE_TYPE_FRINGE: + if (!pmReadoutCombine(outRO, readouts, zeros, scales, combination)) { + psFree(readouts); + psFree(outRO); + goto ERROR; + } + break; + default: + psAbort("Should never get here."); + } + + + for (int i = 0; i < numFiles && more; i++) { + pmReadout *inRO = readouts->data[i]; + + // Check to see if there's more chunks to read + #define MORE_CHUNK(NAME,TYPE) { \ + pmFPAfile *file = pmFPAfileSelectSingle(config->files, NAME, i); \ + more &= pmReadoutMore##TYPE(inRO, file->fits, 0, rows); \ + } + + MORE_CHUNK("PPMERGE.INPUT", /* Blank */); + if (haveMasks) { + MORE_CHUNK("PPMERGE.INPUT.MASK", Mask); + } + if (haveWeights) { + MORE_CHUNK("PPMERGE.INPUT.WEIGHT", Weight); + } + } + } + + // Get list of cells for concepts averaging + psList *inCells = psListAlloc(NULL); // List of cells + for (int i = 0; i < numFiles; i++) { + pmReadout *readout = readouts->data[i]; // Readout of interest + psListAdd(inCells, PS_LIST_TAIL, readout->parent); + } + if (!pmConceptsAverageCells(outCell, inCells, NULL, NULL, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average cell concepts."); + psFree(inCells); + psFree(outRO); + goto ERROR; + } + psFree(inCells); + + psFree(readouts); + + // Plug supplementary images into their own FPAs + { + pmCell *countsCell = pmFPAfileThisCell(config->files, view, "PPMERGE.OUTPUT.COUNT"); + pmReadout *countsRO = pmReadoutAlloc(countsCell); // Readout with count of inputs per pixel + psImage *counts = psMetadataLookupPtr(NULL, outRO->analysis, PM_READOUT_STACK_ANALYSIS_COUNT); + countsRO->image = psImageCopy(countsRO->image, counts, PS_TYPE_F32); + psMetadataRemoveKey(outRO->analysis, PM_READOUT_STACK_ANALYSIS_COUNT); + countsRO->data_exists = countsCell->data_exists = countsCell->parent->data_exists = true; + psFree(countsRO); + + pmCell *sigmaCell = pmFPAfileThisCell(config->files, view, "PPMERGE.OUTPUT.SIGMA"); + pmReadout *sigmaRO = pmReadoutAlloc(sigmaCell); // Readout with stdev per pixel + psImage *sigma = psMetadataLookupPtr(NULL, outRO->analysis, PM_READOUT_STACK_ANALYSIS_SIGMA); + sigmaRO->image = psImageCopy(sigmaRO->image, sigma, PS_TYPE_F32); + psMetadataRemoveKey(outRO->analysis, PM_READOUT_STACK_ANALYSIS_SIGMA); + sigmaRO->data_exists = sigmaCell->data_exists = sigmaCell->parent->data_exists = true; + psFree(sigmaRO); + } + + // Measure the fringes for this cell + // + // XXX Need to deal with multiple components: we will do this by building up the components + // Read the existing fringe measurements + // Use existing regions to measure fringe statistics + // Add the new fringe measurements to the existing fringe measurements. + // Write the appended fringe measurements. + // Read in the "output" file to get the existing components. + // Put the new readout into the cell after the existing readouts. + if (type == PPMERGE_TYPE_FRINGE && outRO) { + pmFringeRegions *regions = pmFringeRegionsAlloc(fringeNum, fringeSize, fringeSize, + fringeSmoothX, fringeSmoothY); + pmFringeStats *fringe = pmFringeStatsMeasure(regions, outRO, maskVal); + psFree(regions); + if (!fringe) { + psError(PS_ERR_UNKNOWN, false, "Unable to measure fringe statistics.\n"); + psFree(outRO); + goto ERROR; + } + + psArray *fringes = psArrayAlloc(1); // Array of fringes + fringes->data[0] = fringe; + + pmFringesFormat(outCell, NULL, fringes); + psFree(fringes); // Drop reference + } + + if (!ppStatsFPA(stats, outFPA, view, maskVal | pmConfigMask("BLANK", config), config)) { + psError(PS_ERR_UNKNOWN, true, "Unable to generate stats for image."); + goto ERROR; + } + + psFree(outRO); + cellNum++; + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + goto ERROR; + } + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + goto ERROR; + } + } + + // Get list of FPAs for concepts averaging + psList *inFPAs = psListAlloc(NULL); // List of FPAs + for (int i = 0; i < numFiles; i++) { + pmFPAfile *input = inputs->data[i]; // Input file + psListAdd(inFPAs, PS_LIST_TAIL, input->fpa); + } + if (!pmConceptsAverageFPAs(output->fpa, inFPAs)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts."); + psFree(inFPAs); + goto ERROR; + } + psFree(inFPAs); + + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + goto ERROR; + } + + psFree(view); + psFree(combination); + psFree(inputs); + psFree(masks); + psFree(weights); + psFree(stats); + return true; + +ERROR: + psFree(view); + psFree(combination); + psFree(inputs); + psFree(masks); + psFree(weights); + psFree(stats); + return false; +} + Index: /tags/pap_merge_030828/ppMerge/src/ppMergeMask.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeMask.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeMask.c (revision 17232) @@ -0,0 +1,405 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include +#include +#include + +#include "ppMerge.h" + +static bool mergeMask(pmConfig *config, // Configuration + const pmFPAview *view, // View to chip + bool writeOut, // Write output? + psRandom *rng, // Random number generator + psMetadata *stats // Statistics output + ) +{ + assert(config); + assert(view); + assert(view->chip != -1 && view->cell == -1 && view->readout == -1); + + bool mdok; // Status of MD lookup + int numFiles = psMetadataLookupS32(NULL, config->arguments, "INPUTS.NUM"); // Number of input files + psStatsOptions meanStat = psMetadataLookupS32(NULL, config->arguments, "MEAN"); // Statistic for mean + psStatsOptions stdevStat = psMetadataLookupS32(NULL, config->arguments, "STDEV"); // Statistic for stdev + psMaskType maskVal = psMetadataLookupU8(NULL, config->arguments, "MASKVAL"); // Value to mask + int sample = psMetadataLookupS32(NULL, config->arguments, "SAMPLE"); // Size of sample for statistics + bool chipStats = psMetadataLookupBool(&mdok, config->arguments, "MASK.CHIPSTATS"); // Statistics on chip? + float maskSuspect = psMetadataLookupF32(NULL, config->arguments, "MASK.SUSPECT"); // Threshold for suspect pixels + float maskBad = psMetadataLookupF32(NULL, config->arguments, "MASK.BAD"); // Threshold for bad pixels + pmMaskIdentifyMode maskMode = psMetadataLookupS32(NULL, config->arguments, "MASK.MODE"); // Mode for identifying bad pixels + int maskGrow = psMetadataLookupS32(NULL, config->arguments, "MASK.GROW"); // Radius to grow mask + psMaskType maskGrowVal = psMetadataLookupU8(NULL, config->arguments, "MASK.GROWVAL"); // Value for grown mask + + psStats *statistics = psStatsAlloc(meanStat | stdevStat); // Statistics for background + + psString outName = ppMergeOutputFile(config); // Name of output file + pmChip *outChip = pmFPAfileThisChip(config->files, view, outName); // Output chip + psFree(outName); + + // For each input file, get the statistics, which can be calculated at the chip or cell levels + psVector *values = psVectorAlloc(sample, PS_TYPE_F32); // Pixel values for statistics + pmFPAview *inView = pmFPAviewAlloc(0); // View for input + for (int i = 0; i < numFiles; i++) { + pmFPAfileActivate(config->files, false, NULL); + psArray *files = ppMergeFileActivateSingle(config, PPMERGE_FILES_INPUT, true, i); // Input files + pmFPAfile *input = files->data[0]; // Input file + psFree(files); + pmFPA *inFPA = input->fpa; // Input FPA + *inView = *view; + + int valueIndex = 0; // Index for vector of pixel values + + pmCell *inCell; // Input cell + while ((inCell = pmFPAviewNextCell(inView, inFPA, 1))) { + pmHDU *hdu = pmHDUFromCell(inCell); // HDU for cell + if (!hdu || hdu->blankPHU) { + // No data here + continue; + } + psTrace("ppMerge", 1, "Getting suspect pixels for file %d chip %d cell %d", + i, inView->chip, inView->cell); + + if (!pmFPAfileIOChecks(config, inView, PM_FPA_BEFORE)) { + psFree(inView); + goto MERGE_MASK_ERROR; + } + + if (!ppMergeFileOpenInput(config, view, i)) { + psError(PS_ERR_UNKNOWN, false, "Unable to open file %d", i); + psFree(inView); + goto MERGE_MASK_ERROR; + } + + if (inCell->readouts->n > 1) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "File %d chip %d cell %d contains more than one readout (%ld)", + i, inView->chip, inView->cell, inCell->readouts->n); + psFree(inView); + goto MERGE_MASK_ERROR; + } + + pmReadout *readout; + if (inCell->readouts && inCell->readouts->n == 1) { + readout = psMemIncrRefCounter(inCell->readouts->data[0]); // Input readout + } else { + readout = pmReadoutAlloc(inCell); + } + + if (!ppMergeFileReadInput(config, readout, i, 0)) { + psError(PS_ERR_UNKNOWN, false, "Unable to read readout %d", i); + psFree(inView); + psFree(readout); + goto MERGE_MASK_ERROR; + } + + pmCell *outCell = pmFPAfileThisCell(config->files, inView, "PPMERGE.OUTPUT.MASK"); // Output cell + pmReadout *outRO = NULL; // Output readout + if (outCell->readouts && outCell->readouts->n == 1) { + outRO = psMemIncrRefCounter(outCell->readouts->data[0]); + } else { + outRO = pmReadoutAlloc(outCell); + } + psImage *outMask = outRO->mask; // Output mask image (for iterative generation of mask) + + psImage *image = readout->image, *mask = readout->mask; // Image and mask + int numCols = readout->image->numCols, numRows = readout->image->numRows; // Image size + int numPix = numCols * numRows; // Number of pixels + int numCells = chipStats ? readout->parent->parent->cells->n : 1; // Number of cells + int num = PS_MIN(numPix, sample / numCells); // Number of values to add + if (!chipStats) { + valueIndex = 0; + } + for (int i = 0; i < num; i++) { + int pixel = numPix * psRandomUniform(rng); + int x = pixel % numCols; + int y = pixel / numCols; + if ((mask && (mask->data.PS_TYPE_MASK_DATA[y][x] & maskVal)) || + !isfinite(image->data.F32[y][x]) || + (outMask && (outMask->data.PS_TYPE_MASK_DATA[y][x] & maskVal))) { + continue; + } + + values->data.F32[valueIndex++] = image->data.F32[y][x]; + } + + if (!chipStats) { + values->n = valueIndex; + if (!psVectorStats(statistics, values, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "Unable to do statistics on readout."); + psFree(inView); + psFree(readout); + goto MERGE_MASK_ERROR; + } + + if (!pmMaskFlagSuspectPixels(outRO, readout, psStatsGetValue(statistics, meanStat), + psStatsGetValue(statistics, stdevStat), maskSuspect, maskVal)) { + psError(PS_ERR_UNKNOWN, false, "Unable to find suspect values in file %d", i); + psFree(inView); + psFree(readout); + goto MERGE_MASK_ERROR; + } + pmCellFreeData(inCell); + + if (!pmFPAfileIOChecks(config, inView, PM_FPA_AFTER)) { + psFree(inView); + psFree(readout); + goto MERGE_MASK_ERROR; + } + } + psFree(readout); + psFree(outRO); + } + + // Additional run through cells if we want chip-level statistics + if (chipStats && valueIndex > 0) { + values->n = valueIndex; + if (!psVectorStats(statistics, values, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "Unable to do statistics on chip."); + goto MERGE_MASK_ERROR; + } + inView->cell = -1; + while ((inCell = pmFPAviewNextCell(inView, inFPA, 1))) { + pmHDU *hdu = pmHDUFromCell(inCell); // HDU for cell + if (!hdu || hdu->blankPHU) { + // No data here + continue; + } + pmReadout *readout = inCell->readouts->data[0]; // Readout of interest + + inView->readout = 0; + pmReadout *outRO = pmFPAfileThisReadout(config->files, inView, "PPMERGE.OUTPUT.MASK"); + + if (!pmMaskFlagSuspectPixels(outRO, readout, psStatsGetValue(statistics, meanStat), + psStatsGetValue(statistics, stdevStat), maskSuspect, maskVal)) { + psError(PS_ERR_UNKNOWN, false, "Unable to find suspect values in file %d", i); + goto MERGE_MASK_ERROR; + } + pmCellFreeData(inCell); + + inView->readout = -1; + if (!pmFPAfileIOChecks(config, inView, PM_FPA_AFTER)) { + psFree(inView); + goto MERGE_MASK_ERROR; + } + } + } + } + psFree(inView); + psFree(statistics); statistics = NULL; + psFree(values); values = NULL; + + + // Another run through the chip to threshold on the suspects + pmFPAfileActivate(config->files, false, NULL); + if (writeOut) { + ppMergeFileActivate(config, PPMERGE_FILES_OUTPUT, true); + } + pmFPA *outFPA = outChip->parent; // Output FPA + pmCell *outCell; // Output cell + pmFPAview *outView = pmFPAviewAlloc(0); // View into output FPA + *outView = *view; + while ((outCell = pmFPAviewNextCell(outView, outFPA, 1))) { + pmHDU *hdu = pmHDUFromCell(outCell); // HDU for cell + if (!hdu || hdu->blankPHU) { + // No data here + continue; + } + + psTrace("ppMerge", 1, "Getting bad pixels for chip %d cell %d", outView->chip, outView->cell); + + assert(outCell->readouts && outCell->readouts->n == 1); + pmReadout *outRO = outCell->readouts->data[0]; // Output readout + if (!pmMaskIdentifyBadPixels(outRO, maskVal, maskBad, maskMode)) { + psError(PS_ERR_UNKNOWN, false, "Unable to mask bad pixels"); + goto MERGE_MASK_ERROR; + } + + // Supplementary outputs + { + // The counts image is fairly useless, but it preserves the model + pmCell *countsCell = pmFPAfileThisCell(config->files, outView, "PPMERGE.OUTPUT.COUNT"); + pmReadout *countsRO = pmReadoutAlloc(countsCell); // Readout with count of inputs per pixel + countsRO->image = psImageAlloc(outRO->mask->numCols, outRO->mask->numRows, PS_TYPE_F32); + psImageInit(countsRO->image, numFiles); + countsRO->data_exists = countsCell->data_exists = countsCell->parent->data_exists = true; + psFree(countsRO); + + pmCell *sigmaCell = pmFPAfileThisCell(config->files, outView, "PPMERGE.OUTPUT.SIGMA"); + pmReadout *sigmaRO = pmReadoutAlloc(sigmaCell); // Readout with suspect image + psImage *suspect = psMetadataLookupPtr(NULL, outRO->analysis, PM_MASK_ANALYSIS_SUSPECT); + sigmaRO->image = psImageCopy(sigmaRO->image, suspect, PS_TYPE_F32); + psMetadataRemoveKey(outRO->analysis, PM_MASK_ANALYSIS_SUSPECT); + sigmaRO->data_exists = sigmaCell->data_exists = sigmaCell->parent->data_exists = true; + psFree(sigmaRO); + } + + if (maskGrowVal > 0) { + psImage *grown = psImageGrowMask(NULL, outRO->mask, maskVal, maskGrow, maskGrowVal); // Grown mask + psFree(outRO->mask); + outRO->mask = grown; + } + + if (writeOut) { + if (!pmFPAfileIOChecks(config, outView, PM_FPA_BEFORE)) { + psFree(outView); + goto MERGE_MASK_ERROR; + } + + outRO->data_exists = outCell->data_exists = outChip->data_exists = true; + + // Average concepts + psList *cells = psListAlloc(NULL); // List of cells, for concept averaging + for (int i = 0; i < numFiles; i++) { + pmFPAfile *inFile = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT", i); // Input file + pmCell *inCell = pmFPAviewThisCell(outView, inFile->fpa); // Input cell + psListAdd(cells, PS_LIST_TAIL, inCell); + } + if (!pmConceptsAverageCells(outCell, cells, NULL, NULL, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average cell concepts."); + psFree(cells); + psFree(outRO); + psFree(outView); + return false; + } + psFree(cells); + + // Statistics on the merged cell using a fake image + outRO->image = psImageAlloc(outRO->mask->numCols, outRO->mask->numRows, PS_TYPE_F32); + psImageInit(outRO->image, 1.0); + if (!ppStatsFPA(stats, outRO->parent->parent->parent, outView, + maskVal | pmConfigMask("BLANK", config), config)) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to generate stats for image."); + psFree(outRO); + psFree(outView); + return false; + } + psFree(outRO->image); + outRO->image = NULL; + + // Write + if (!pmFPAfileIOChecks(config, outView, PM_FPA_AFTER)) { + psFree(outView); + goto MERGE_MASK_ERROR; + } + } + } + psFree(outView); + + ppMergeFileActivate(config, PPMERGE_FILES_ALL, true); + + return true; + + +MERGE_MASK_ERROR: + psFree(statistics); + psFree(values); + return false; +} + + + + + +bool ppMergeMask(pmConfig *config) +{ + assert(config); + + bool mdok; // Status of MD lookup + int numFiles = psMetadataLookupS32(NULL, config->arguments, "INPUTS.NUM"); // Number of inputs + bool haveMasks = psMetadataLookupBool(&mdok, config->arguments, "INPUTS.MASKS"); // Do we have masks? + bool haveWeights = psMetadataLookupBool(&mdok, config->arguments, "INPUTS.WEIGHTS"); // Do we have weights? + int iter = psMetadataLookupS32(NULL, config->arguments, "ITER"); // Number of rejection iterations + + PS_ASSERT_INT_POSITIVE(iter, false); + + pmFPAview *view = pmFPAviewAlloc(0); // View to component of interest + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator + + psMetadata *stats = NULL; // Statistics for output + if (psMetadataLookup(config->arguments, "STATS.NAME")) { + stats = psMetadataAlloc(); + psMetadataAddMetadata(config->arguments, PS_LIST_TAIL, "STATS.DATA", 0, "Statistics output", stats); + } + + psArray *inputs = ppMergeFileDataLevel(config, "PPMERGE.INPUT", PM_FPA_LEVEL_READOUT); // Input images + psFree(inputs); + if (haveMasks) { + psArray *masks = ppMergeFileDataLevel(config, "PPMERGE.INPUT.MASK", PM_FPA_LEVEL_READOUT); + psFree(masks); + } + if (haveWeights) { + psArray *weights = ppMergeFileDataLevel(config, "PPMERGE.INPUT.WEIGHT", PM_FPA_LEVEL_READOUT); + psFree(weights); + } + + if (!ppMergeFileActivate(config, PPMERGE_FILES_INPUT, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to activate files."); + goto PPMERGE_MASK_ERROR; + } + if (!ppMergeFileActivate(config, PPMERGE_FILES_OUTPUT, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to activate files."); + goto PPMERGE_MASK_ERROR; + } + + psString outName = ppMergeOutputFile(config); // Name of output file + pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, outName); // Output file + psFree(outName); + assert(output && output->fpa); + pmFPA *outFPA = output->fpa; // Output FPA + + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + goto PPMERGE_MASK_ERROR; + } + pmChip *outChip; // Chip of interest + while ((outChip = pmFPAviewNextChip(view, outFPA, 1))) { + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + goto PPMERGE_MASK_ERROR; + } + + for (int i = 0; i < iter; i++) { + if (!mergeMask(config, view, (i == iter - 1), rng, stats)) { + psError(PS_ERR_UNKNOWN, false, "Unable to merge chip %d", view->chip); + goto PPMERGE_MASK_ERROR; + } + } + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + goto PPMERGE_MASK_ERROR; + } + } + + psList *fpaList = psListAlloc(NULL);// List of FPAs for concept averaging + for (int i = 0; i < numFiles; i++) { + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPMERGE.INPUT", i); // Input file + psListAdd(fpaList, PS_LIST_TAIL, file->fpa); + } + if (!pmConceptsAverageFPAs(outFPA, fpaList)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts."); + psFree(fpaList); + goto PPMERGE_MASK_ERROR; + } + psFree(fpaList); + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + goto PPMERGE_MASK_ERROR; + } + + psFree(stats); + psFree(view); + psFree(rng); + + return true; + +PPMERGE_MASK_ERROR: + psFree(stats); + psFree(view); + psFree(rng); + return false; +} + Index: /tags/pap_merge_030828/ppMerge/src/ppMergeMask.h =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeMask.h (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeMask.h (revision 17232) @@ -0,0 +1,50 @@ +#ifndef PP_MERGE_MASK +#define PP_MERGE_MASK + +#include +#include "ppMergeData.h" +#include "ppMergeOptions.h" + +// Generate a mask +bool ppMergeMask(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +bool ppMergeMaskSuspect(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +bool ppMergeMaskBad(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +bool ppMergeMaskAverageConcepts(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +bool ppMergeMaskWrite(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +bool ppMergeMaskGrow(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ); + +bool ppMergeMaskGrowReadout (pmReadout *readout, psMaskType maskVal, int nPixels); + +bool ppMergeMaskReadoutStats(const pmReadout *readout, + const pmReadout *output, + ppMergeOptions *options, // Options + psRandom *rng); + +bool ppMergeMaskChipStats (const pmChip *chip, + const pmChip *output, + ppMergeOptions *options, + psRandom *rng); +#endif Index: /tags/pap_merge_030828/ppMerge/src/ppMergeMaskAverageConcepts.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeMaskAverageConcepts.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeMaskAverageConcepts.c (revision 17232) @@ -0,0 +1,83 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask +bool ppMergeMaskAverageConcepts(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + pmFPA *fpaOut = data->out; // Output FPA + + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + if (fpaOut->hdu) { + pmFPAUpdateNames(fpaOut, NULL, NULL); + } + + pmChip *chipOut; // Output chip of interest + while ((chipOut = pmFPAviewNextChip(view, fpaOut, 1))) { + + pmCell *cellOut; // Output cell of interest + while ((cellOut = pmFPAviewNextCell(view, fpaOut, 1))) { + + pmHDU *hdu = pmHDUGetLowest(fpaOut, chipOut, cellOut); + if (!hdu || hdu->blankPHU) { + continue; + } + + // Get list of cells for concepts averaging + psList *inCells = psListAlloc(NULL); // List of cells + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmCell *cellIn = pmFPAviewThisCell(view, data->in->data[i]); // Input cell + psListAdd(inCells, PS_LIST_TAIL, cellIn); + } + if (!pmConceptsAverageCells(cellOut, inCells, NULL, NULL, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average cell concepts."); + psFree(inCells); + return false; + } + psFree(inCells); + } + } + + // Get list of FPAs for concepts averaging + psList *inFPAs = psListAlloc(NULL); // List of FPAs + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmFPA *fpaIn = data->in->data[i]; // Input FPA + psListAdd(inFPAs, PS_LIST_TAIL, fpaIn); + } + + if (!pmConceptsAverageFPAs(fpaOut, inFPAs)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts."); + psFree(inFPAs); + return false; + } + psFree(inFPAs); + + psFree(view); + + return true; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeMaskBad.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeMaskBad.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeMaskBad.c (revision 17232) @@ -0,0 +1,83 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask +bool ppMergeMaskBad(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + pmFPA *fpaOut = data->out; // Output FPA + + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + + pmChip *chipOut; // Output chip of interest + while ((chipOut = pmFPAviewNextChip(view, fpaOut, 1))) { + + pmCell *cellOut; // Output cell of interest + while ((cellOut = pmFPAviewNextCell(view, fpaOut, 1))) { + + psImage *suspect = psMetadataLookupPtr(NULL, cellOut->analysis, "MASK.SUSPECT"); + if (! suspect) { + continue; + } + + pmReadout *roOut = NULL; + if (cellOut->readouts->n > 0) { + roOut = psMemIncrRefCounter (cellOut->readouts->data[0]); + psFree (roOut->mask); + psTrace("ppMerge", 5, "save results with mask from previous pass\n"); + } else { + roOut = pmReadoutAlloc(cellOut); // Output readout + } + + roOut->mask = pmMaskIdentifyBadPixels(suspect, options->combine->maskVal, filenames->n, options->maskBad, options->maskMode); + roOut->data_exists = cellOut->data_exists = chipOut->data_exists = true; + + // Statistics on the merged cell + if (data->statsFile) { + if (!data->stats) { + data->stats = psMetadataAlloc(); + } + + // Build a fake image to do statistics + roOut->image = psImageAlloc(roOut->mask->numCols, roOut->mask->numRows, PS_TYPE_F32); + psImageInit(roOut->image, 1.0); + if (!ppStatsFPA(data->stats, data->out, view, + options->combine->maskVal | pmConfigMask("BLANK", config), + config)) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to generate stats for image.\n"); + return false; + } + psFree(roOut->image); + roOut->image = NULL; + } + psFree(roOut); // Drop reference + + // drop this reference (unless??) + psMetadataRemoveKey (cellOut->analysis, "MASK.SUSPECT"); + } + } + + psFree(view); + + return true; +} + Index: /tags/pap_merge_030828/ppMerge/src/ppMergeMaskByImageStats.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeMaskByImageStats.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeMaskByImageStats.c (revision 17232) @@ -0,0 +1,87 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask image consisting of pixels which are outliers from the image mean +bool ppMergeMaskByImageStats(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator + pmFPA *fpaOut = data->out; // Output FPA + + // Iterate over each file + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + psFits *fits = data->files->data[i]; // FITS file handle + if (!fits) { + continue; + } + psTrace("ppMerge", 3, "File %d: %s\n", i, (const char*)filenames->data[i]); + + pmFPA *fpaIn = data->in->data[i]; // Input FPA + pmFPAview *view = pmFPAviewAlloc(0); // View of FPA, for iteration + pmChip *chipIn; // Input chip of interest + while ((chipIn = pmFPAviewNextChip(view, fpaIn, 1))) { + pmCell *cellIn; // Input cell of interest + while ((cellIn = pmFPAviewNextCell(view, fpaIn, 1))) { + if (!pmCellRead(cellIn, fits, config->database)) { + continue; + } + if (cellIn->readouts->n == 0) { + continue; + } + + pmCell *cellOut = pmFPAviewThisCell(view, fpaOut); // Output cell + // Suspect pixels image + psImage *suspect = psMetadataLookupPtr(NULL, cellOut->analysis, "MASK.SUSPECT"); + bool first = suspect ? false : true; + + pmReadout *roIn; // Input readout of interest + while ((roIn = pmFPAviewNextReadout(view, fpaIn, 1))) { + if (!roIn->image) { + continue; + } + psTrace("ppMerge", 4, "Flagging suspect pixels in chip %d, cell %d, ro %d\n", + view->chip, view->cell, view->readout); + float frac = options->sample / (float)(roIn->image->numCols * roIn->image->numRows); + suspect = pmMaskFlagSuspectPixels(suspect, roIn, options->maskSuspect, + options->combine->maskVal, frac, rng); + } + + if (first) { + psMetadataAddImage(cellOut->analysis, PS_LIST_TAIL, "MASK.SUSPECT", 0, + "Suspect pixels", suspect); + psFree(suspect); + } + + pmCellFreeData(cellIn); + } + pmChipFreeData(chipIn); + } + pmFPAFreeData(fpaIn); + psFree(view); + } + psFree(rng); + + return true; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeMaskByPixelStats.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeMaskByPixelStats.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeMaskByPixelStats.c (revision 17232) @@ -0,0 +1,177 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeCombine.h" +#include "ppMergeVersion.h" + +// XXX for the moment, this function and ppMergeMaskByImageStats both load (and free) the input +// data + +// Combine the inputs +bool ppMergeMaskByPixelStats(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + // Iterate over the FPA + pmFPA *fpa = data->out; // Output FPA + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + int cellNum = -1; // Cell number in the whole FPA + if (data->out->hdu) { + pmFPAUpdateNames(data->out, NULL, NULL); + } + + pmChip *chip; // Chip of interest + pmHDU *lastHDU = NULL; // Last HDU to be updated + + while ((chip = pmFPAviewNextChip(view, fpa, 1))) { + if (chip->hdu) { + // Data will exist soon + pmFPAUpdateNames(data->out, chip, NULL); + chip->data_exists = true; + } + pmCell *cell; // Cell of interest + while ((cell = pmFPAviewNextCell(view, fpa, 1))) { + cellNum++; + if (cell->hdu) { + // Data will exist soon + pmFPAUpdateNames(data->out, chip, cell); + chip->data_exists = cell->data_exists = true; + } + pmReadout *readout = pmReadoutAlloc(cell); // Output readout of interest + psArray *stack = psArrayAlloc(filenames->n); // Stack of readouts to combine + + // Read bit by bit + int numRead; // Number of inputs read + int numScan = 0; + + // Put version metadata into header + pmHDU *hdu = pmHDUFromCell(cell); + if (hdu && hdu != lastHDU) { + if (!hdu->header) { + hdu->header = psMetadataAlloc(); + } + ppMergeVersionMetadata(hdu->header); + lastHDU = hdu; + } + + while (1) { + numRead = 0; + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + psFits *fits = data->files->data[i]; // FITS file handle + if (!fits) { + continue; + } + + if (!stack->data[i]) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + pmChip *chipIn = fpaIn->chips->data[view->chip]; // Input chip + pmCell *cellIn = chipIn->cells->data[view->cell]; // Input cell + stack->data[i] = pmReadoutAlloc(cellIn); // Input readout + } + + // Only reading and writing the first readout in each cell (plane 0) + bool readOK; + if (pmReadoutReadNext(&readOK, stack->data[i], fits, 0, options->rows)) { + if (!readOK) { + psError(PS_ERR_IO, false, "Failed to read concepts for cell.\n"); + psErrorStackPrint(stderr, "trouble reading data!\n"); + exit (1); + } + numRead++; + } else { + psTrace("ppMerge", 3, "Unable to read from file %d for chip %d, " + "cell %d, scan %d\n", i, view->chip, view->cell, numScan); + } + } + + psTrace("ppMerge", 5, "Chip %d, cell %d, scan %d\n", view->chip, view->cell, numScan); + if (numRead == 0) { + break; + } + ppMergeMaskReadoutByPixelStats(readout, stack, options->combine); + numScan++; + } + + // Get list of cells for concepts averaging + // XXX only do this operation once between Image and Pixel Stats + psList *inCells = psListAlloc(NULL); // List of cells + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmCell *cellIn = pmFPAviewThisCell(view, data->in->data[i]); // Input cell + psListAdd(inCells, PS_LIST_TAIL, cellIn); + } + if (!pmConceptsAverageCells(cell, inCells, NULL, NULL, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average cell concepts."); + psFree(inCells); + return false; + } + psFree(inCells); + psFree(stack); + psFree(readout); // Drop reference + + // Blow away the cell data + for (int i = 0; i < filenames->n; i++) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + if (!fpaIn) { continue; } // was not a valid input file + pmChip *chipIn = fpaIn->chips->data[view->chip]; // Input chip + pmCell *cellIn = chipIn->cells->data[view->cell]; // Input cell + pmCellFreeData(cellIn); + } + } + + // Blow away the chip data + for (int i = 0; i < filenames->n; i++) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + if (!fpaIn) { continue; } // was not a valid input file + pmChip *chipIn = fpaIn->chips->data[view->chip]; // Input chip + pmChipFreeData(chipIn); + } + } + + // Blow away the FPA data + for (int i = 0; i < filenames->n; i++) { + pmFPA *fpaIn = data->in->data[i]; // Input FPA + if (!fpaIn) { continue; } // was not a valid input file + pmFPAFreeData(fpaIn); + } + + psFree(view); + + // Get list of FPAs for concepts averaging + psList *inFPAs = psListAlloc(NULL); // List of FPAs + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmFPA *fpaIn = data->in->data[i]; // Input FPA + psListAdd(inFPAs, PS_LIST_TAIL, fpaIn); + } + if (!pmConceptsAverageFPAs(fpa, inFPAs)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts."); + psFree(inFPAs); + return false; + } + psFree(inFPAs); + + return true; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeMaskGrow.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeMaskGrow.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeMaskGrow.c (revision 17232) @@ -0,0 +1,89 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask +bool ppMergeMaskGrow(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + pmFPA *fpaOut = data->out; // Output FPA + + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + + pmChip *chipOut; // Output chip of interest + while ((chipOut = pmFPAviewNextChip(view, fpaOut, 1))) { + + pmCell *cellOut; // Output cell of interest + while ((cellOut = pmFPAviewNextCell(view, fpaOut, 1))) { + + pmReadout *roOut; + while ((roOut = pmFPAviewNextReadout(view, fpaOut, 1))) { + + // do this N iterations (XXX add to options) + ppMergeMaskGrowReadout(roOut, options->combine->maskVal, options->growPixels); + ppMergeMaskGrowReadout(roOut, options->combine->maskVal, options->growPixels); + } + } + } + + psFree(view); + + return true; +} + +bool ppMergeMaskGrowReadout (pmReadout *readout, psMaskType maskVal, int nPixels) { + + if (!readout->mask) return true; + + psImage *oldMask = readout->mask; + psImage *newMask = psImageCopy (NULL, oldMask, PS_TYPE_U8); + + psImageInit (newMask, 0); + + int Nx = oldMask->numCols; + int Ny = oldMask->numRows; + + for (int iy = 0; iy < Ny; iy++) { + for (int ix = 0; ix < Nx; ix++) { + int nSum = 0; + for (int jy = -1; jy <= +1; jy++) { + int ny = iy + jy; + if (ny < 0) continue; + if (ny >= Ny) continue; + for (int jx = -1; jx <= +1; jx++) { + if (!jx && !jy) continue; + int nx = ix + jx; + if (nx < 0) continue; + if (nx >= Nx) continue; + if (oldMask->data.U8[ny][nx] & maskVal) { + nSum ++; + } + } + } + if (nSum >= nPixels) { + newMask->data.U8[iy][ix] = maskVal; + } else { + newMask->data.U8[iy][ix] = oldMask->data.U8[iy][ix]; + } + } + } + psFree (readout->mask); + readout->mask = newMask; + + return true; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeMaskOutput.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeMaskOutput.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeMaskOutput.c (revision 17232) @@ -0,0 +1,157 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// write out the mask: assumes it has been saved on the cell->analysis as MASK.UNION +bool ppMergeMaskOutput(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + pmFPA *fpaOut = data->out; // Output FPA + + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + if (fpaOut->hdu) { + pmFPAUpdateNames(fpaOut, NULL, NULL); + } + pmFPAWriteMask(fpaOut, data->outFile, config->database, true, false); // Write header only + pmChip *chipOut; // Output chip of interest + while ((chipOut = pmFPAviewNextChip(view, fpaOut, 1))) { + if (chipOut->hdu) { + chipOut->data_exists = true; + pmFPAUpdateNames(fpaOut, chipOut, NULL); + } + pmChipWriteMask(chipOut, data->outFile, config->database, true, false); // Write header only + pmCell *cellOut; // Output cell of interest + while ((cellOut = pmFPAviewNextCell(view, fpaOut, 1))) { + if (cellOut->hdu) { + chipOut->data_exists = cellOut->data_exists = true; + pmFPAUpdateNames(fpaOut, chipOut, cellOut); + } + pmCellWriteMask(cellOut, data->outFile, config->database, true); // Write header only + + psImage *maskUnion = psMetadataLookupPtr(NULL, cellOut->analysis, "MASK.UNION"); + if (! maskUnion) { + continue; + } + + pmReadout *roOut = pmReadoutAlloc(cellOut); // Output readout + roOut->mask = maskUnion; + roOut->data_exists = cellOut->data_exists = chipOut->data_exists = true; + + // XXX skip this? Get list of cells for concepts averaging + { + psList *inCells = psListAlloc(NULL); // List of cells + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmCell *cellIn = pmFPAviewThisCell(view, data->in->data[i]); // Input cell + psListAdd(inCells, PS_LIST_TAIL, cellIn); + } + if (!pmConceptsAverageCells(cellOut, inCells, NULL, NULL, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average cell concepts."); + psFree(inCells); + return false; + } + psFree(inCells); + } + + { + // Add MD5 information for cell + pmHDU *hdu = pmHDUFromCell(cell); // HDU that owns the cell + const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME"); + const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME"); + + psString headerName = NULL; // Header name for MD5 + psStringAppend(&headerName, "MD5_%s_%s", chipName, cellName); + + psVector *md5 = psImageMD5(readout->image); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, headerName, PS_META_REPLACE, + "Image MD5", md5string); + psFree(md5string); + psFree(headerName); + } + + // Statistics on the merged cell + if (data->statsFile) { + if (!data->stats) { + data->stats = psMetadataAlloc(); + } + + // Build a fake image to do statistics + roOut->image = psImageAlloc(roOut->mask->numCols, roOut->mask->numRows, PS_TYPE_F32); + psImageInit(roOut->image, 1.0); + if (!ppStatsFPA(data->stats, data->out, view, + options->combine->maskVal | pmConfigMask("BLANK", config), + config)) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to generate stats for image.\n"); + return false; + } + psFree(roOut->image); + roOut->image = NULL; + } + + psFree(roOut); // Drop reference + + if (cellOut->hdu && !cellOut->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out cell HDU.\n"); + pmCellWriteMask(cellOut, data->outFile, config->database, false); + pmCellFreeData(cellOut); + } + } + + if (chipOut->hdu && !chipOut->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out chip HDU.\n"); + pmChipWriteMask(chipOut, data->outFile, config->database, false, false); + pmChipFreeData(chipOut); + } + } + + // Get list of FPAs for concepts averaging + { + psList *inFPAs = psListAlloc(NULL); // List of FPAs + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + pmFPA *fpaIn = data->in->data[i]; // Input FPA + psListAdd(inFPAs, PS_LIST_TAIL, fpaIn); + } + + if (!pmConceptsAverageFPAs(fpaOut, inFPAs)) { + psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts."); + psFree(inFPAs); + return false; + } + psFree(inFPAs); + } + + if (fpaOut->hdu && !fpaOut->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out FPA HDU.\n"); + pmFPAWriteMask(fpaOut, data->outFile, config->database, false, false); + } + pmFPAFreeData(fpaOut); + + psFree(view); + + return true; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeMaskReadoutByPixelStats.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeMaskReadoutByPixelStats.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeMaskReadoutByPixelStats.c (revision 17232) @@ -0,0 +1,327 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include + +// XXX: Maybe add support for S16 and S32 types. Currently, only F32 supported. +bool ppMergeMaskReadoutByPixelStats(pmReadout *output, const psArray *inputs, const pmCombineParams *params) +{ + // Check inputs + PS_ASSERT_PTR_NON_NULL(output, false); + PS_ASSERT_ARRAY_NON_NULL(inputs, false); + PS_ASSERT_PTR_NON_NULL(params, false); + PS_ASSERT_FLOAT_WITHIN_RANGE(params->fracLow, 0.0, 1.0, false); + PS_ASSERT_FLOAT_WITHIN_RANGE(params->fracHigh, 0.0, 1.0, false); + if (params->combine != PS_STAT_SAMPLE_MEAN && params->combine != PS_STAT_SAMPLE_MEDIAN && + params->combine != PS_STAT_ROBUST_MEDIAN && params->combine != PS_STAT_FITTED_MEAN && + params->combine != PS_STAT_CLIPPED_MEAN) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Combination method is not SAMPLE_MEAN, SAMPLE_MEDIAN, " + "ROBUST_MEDIAN, FITTED_MEAN or CLIPPED_MEAN.\n"); + return false; + } + + // XXX is it possible / desired to use the weight in this analysis? + for (int i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + if (params->weights && !readout->weight) { + psError(PS_ERR_UNEXPECTED_NULL, true, + "Rejection based on weights requested, but no weights supplied for image %d.\n", i); + return false; + } + } + + bool first = !output->image; // First pass through? + + pmHDU *hdu = pmHDUFromReadout(output); // Output HDU + if (!hdu) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find HDU for readout.\n"); + return false; + } + + if (first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Combining using statistic: %x", params->combine); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + psStats *stats = psStatsAlloc(params->combine); // The statistics to use in the combination + if (params->combine == PS_STAT_CLIPPED_MEAN) { + stats->clipSigma = params->rej; + stats->clipIter = params->iter; + + if (first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Combination clipping: %d iterations, rejection at %f sigma", + params->iter, params->rej); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + } + + int minInputCols, maxInputCols, minInputRows, maxInputRows; // Smallest and largest values to combine + int xSize, ySize; // Size of the output image + if (!pmReadoutStackValidate(&minInputCols, &maxInputCols, &minInputRows, &maxInputRows, &xSize, &ySize, + inputs)) { + psError(PS_ERR_UNKNOWN, false, "No valid input readouts."); + return false; + } + + pmReadoutUpdateSize(output, minInputCols, minInputRows, xSize, ySize, true); + psTrace("psModules.imcombine", 7, "Output minimum: %d,%d\n", output->col0, output->row0); + + psStatsOptions combineStdev = 0; // Statistics option for weights + if (params->weights) { + + if (!output->weight) { + output->weight = psImageAlloc(xSize, ySize, PS_TYPE_F32); + } + if (output->weight->numCols < xSize || output->weight->numRows < ySize) { + psImage *newWeight = psImageAlloc(xSize, ySize, PS_TYPE_F32); + psImageInit(newWeight, 0.0); + psImageOverlaySection(newWeight, output->weight, output->col0, output->row0, "="); + psFree(output->weight); + output->weight = newWeight; + } + + if (first) { + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + "Using input weights to combine images", ""); + } + + // Get the correct statistics option for weights + switch (params->combine) { + case PS_STAT_SAMPLE_MEAN: + case PS_STAT_SAMPLE_MEDIAN: + combineStdev = PS_STAT_SAMPLE_STDEV; + break; + case PS_STAT_ROBUST_MEDIAN: + combineStdev = PS_STAT_ROBUST_STDEV; + break; + case PS_STAT_FITTED_MEAN: + combineStdev = PS_STAT_FITTED_STDEV; + break; + case PS_STAT_CLIPPED_MEAN: + combineStdev = PS_STAT_CLIPPED_STDEV; + break; + default: + psAbort("Should never get here --- checked params->combine before.\n"); + } + stats->options |= combineStdev; + } + + // We loop through each pixel in the output image. We loop through each input readout. We determine if + // that output pixel is contained in the image from that readout. If so, we save it in psVector pixels. + // If not, we set a mask for that element in pixels. Then, we mask off pixels not between fracLow and + // fracHigh. Then we call the vector stats routine on those pixels/mask. Then we set the output pixel + // value to the result of the stats call. + + psVector *pixels = psVectorAlloc(inputs->n, PS_TYPE_F32); // Stack of pixels + psF32 *pixelsData = pixels->data.F32; // Dereference pixels + + psVector *mask = psVectorAlloc(inputs->n, PS_TYPE_U8); // Mask for stack + psU8 *maskData = mask->data.U8; // Dereference mask + + psVector *weights = NULL; // Stack of weights + psVector *errors = NULL; // Stack of errors (sqrt of variance/weights), for psVectorStats + psF32 *weightsData = NULL; // Dereference weights + if (params->weights) { + weights = psVectorAlloc(inputs->n, PS_TYPE_F32); // Stack of weights + weightsData = weights->data.F32; + } + psVector *index = NULL; // The indices to sort the pixels + + float keepFrac = 1.0 - params->fracLow - params->fracHigh; // Fraction of pixels to keep + if (keepFrac != 1.0 && first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Min/max rejection: %f high, %f low, keep %d", + params->fracHigh, params->fracLow, params->nKeep); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + psMaskType maskVal = params->maskVal; // The mask value + if (maskVal && first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Mask for combination: %x", maskVal); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + #ifndef PS_NO_TRACE + psTrace("psModules.imcombine", 3, "Iterating output: %d --> %d, %d --> %d\n", + minInputCols - output->col0, maxInputCols - output->col0, + minInputRows - output->row0, maxInputRows - output->row0); + if (psTraceGetLevel("psModules.imcombine") >= 3) { + for (int r = 0; r < inputs->n; r++) { + pmReadout *readout = inputs->data[r]; // Input readout + psTrace("psModules.imcombine", 3, "Iterating input %d: %d --> %d, %d --> %d\n", r, + minInputCols - readout->col0, maxInputCols - readout->col0, + minInputRows - readout->row0, maxInputRows - readout->row0); + } + } + #endif + + // Dereference output products + psF32 **outputImage = output->image->data.F32; // Output image + psU8 **outputMask = output->mask->data.U8; // Output mask + psF32 **outputWeight = NULL; // Output weight map + if (output->weight) { + outputWeight = output->weight->data.F32; + } + + psVector *invScale = NULL; // Inverse scale; pre-calculated for efficiency + if (scale) { + invScale = (psVector*)psBinaryOp(NULL, psScalarAlloc(1.0, PS_TYPE_F32), "/", (const psPtr)scale); + } + + for (int i = minInputRows; i < maxInputRows; i++) { + int yOut = i - output->row0; // y position on output readout + #ifdef SHOW_BUSY + + if (psTraceGetLevel("psModules.imcombine") > 9) { + printf("Processing row %d\r", i); + fflush(stdout); + } + #endif + for (int j = minInputCols; j < maxInputCols; j++) { + int xOut = j - output->col0; // x position on output readout + + int numValid = 0; // Number of valid pixels in the stack + memset(maskData, 0, mask->n * sizeof(psU8)); // Reset the mask + for (int r = 0; r < inputs->n; r++) { + pmReadout *readout = inputs->data[r]; // Input readout + int yIn = i - readout->row0; // y position on input readout + int xIn = j - readout->col0; // x position on input readout + psImage *image = readout->image; // The readout image + + #if 0 // This should have been taken care of already: + // Check bounds + if (xIn < 0 || xIn >= image->numCols || yIn < 0 || yIn >= image->numRows) { + continue; + } + #endif + + pixelsData[r] = image->data.F32[yIn][xIn]; + if (!isfinite(pixelsData[r])) { + maskData[r] = 1; + continue; + } + + // Check mask + psImage *roMask = readout->mask; // The mask image + if (roMask && roMask->data.U8[yIn][xIn] & maskVal) { + maskData[r] = 1; + continue; + } + + if (params->weights) { + weightsData[r] = readout->weight->data.F32[yIn][xIn]; + } + + if (zero) { + pixelsData[r] -= zero->data.F32[r]; + } + if (scale) { + pixelsData[r] *= invScale->data.F32[r]; + if (params->weights) { + weightsData[r] *= invScale->data.F32[r] * invScale->data.F32[r]; + } + } + + numValid++; + } + + if (numValid == 0) { + outputMask[yOut][xOut] = params->blank; + outputImage[yOut][xOut] = NAN; + continue; + } + + // Apply fracLow,fracHigh if there are enough pixels + if (numValid * keepFrac >= params->nKeep && keepFrac != 1.0) { + index = psVectorSortIndex(index, pixels); + int numLow = numValid * params->fracLow; // Number of low pixels to clip + int numHigh = numValid * params->fracHigh; // Number of high pixels to clip + // Low pixels + psS32 *indexData = index->data.S32; // Dereference index + for (int k = 0, numMasked = 0; numMasked < numLow && k < index->n; k++) { + // Don't count the ones that are already masked + if (!maskData[indexData[k]]) { + maskData[indexData[k]] = 1; + numMasked++; + } + } + // High pixels + for (int k = pixels->n - 1, numMasked = 0; numMasked < numHigh && k >= 0; k--) { + // Don't count the ones that are already masked + if (! maskData[indexData[k]]) { + maskData[indexData[k]] = 1; + numMasked++; + } + } + } + + // XXXXX this step probably is very expensive : convert errors to variance everywhere? + if (params->weights) { + errors = (psVector*)psUnaryOp(errors, weights, "sqrt"); + } + + // Combination + if (!psVectorStats(stats, pixels, errors, mask, 1)) { + // Can't do much about it, but it's not worth worrying about + psErrorClear(); + outputImage[yOut][xOut] = NAN; + outputMask[yOut][xOut] = params->blank; + if (params->weights) { + outputWeight[yOut][xOut] = NAN; + } + } else { + outputImage[yOut][xOut] = psStatsGetValue(stats, params->combine); + if (!isfinite(outputImage[yOut][xOut])) { + outputMask[yOut][xOut] = params->blank; + } + if (params->weights) { + float stdev = psStatsGetValue(stats, combineStdev); + outputWeight[yOut][xOut] = PS_SQR(stdev); // Variance + // XXXX this is not the correct formal error. + // also, the weighted mean is not obviously the correct thing here + } + } + } + } + #ifdef SHOW_BUSY + if (psTraceGetLevel("psModules.imcombine") > 9) { + printf("\n"); + } + #endif + psFree(index); + psFree(pixels); + psFree(mask); + psFree(weights); + psFree(errors); + psFree(stats); + psFree(invScale); + + // Update the "concepts" + psList *inputCells = psListAlloc(NULL); // List of cells + for (long i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + psListAdd(inputCells, PS_LIST_TAIL, readout->parent); + } + bool success = pmConceptsAverageCells(output->parent, inputCells, NULL, NULL, true); + psFree(inputCells); + + output->data_exists = true; + output->parent->data_exists = true; + output->parent->parent->data_exists = true; + + return success; +} + Index: /tags/pap_merge_030828/ppMerge/src/ppMergeMaskSuspect.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeMaskSuspect.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeMaskSuspect.c (revision 17232) @@ -0,0 +1,117 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask +bool ppMergeMaskSuspect(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + assert(filenames); // It should be here --- it's put here in ppMergeConfig + + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator + pmFPA *fpaOut = data->out; // Output FPA + + // Iterate over each file + for (int i = 0; i < filenames->n; i++) { + if (! filenames->data[i] || strlen(filenames->data[i]) == 0) { + continue; + } + psFits *fits = data->files->data[i]; // FITS file handle + if (!fits) { + continue; + } + psTrace("ppMerge", 3, "File %d: %s\n", i, (const char*)filenames->data[i]); + + pmFPA *fpaIn = data->in->data[i]; // Input FPA + pmFPAview *view = pmFPAviewAlloc(0); // View of FPA, for iteration + pmChip *chipIn; // Input chip of interest + while ((chipIn = pmFPAviewNextChip(view, fpaIn, 1))) { + + // handle chip vs cell statistics & avoid reading the data twice + + // load the data of all cells + pmCell *cellIn; // Input cell of interest + while ((cellIn = pmFPAviewNextCell(view, fpaIn, 1))) { + if (!pmCellRead(cellIn, fits, config->database)) continue; + if (cellIn->readouts->n == 0) continue; + } + + // calculate the readout statistics either for each readout, or across the entire chip + if (options->statsByChip) { + pmChip *chipOut = pmFPAviewThisChip(view, fpaOut); // Output cell + if (!ppMergeMaskChipStats (chipIn, chipOut, options, rng)) { + psAbort ("stats problem"); + } + } else { + // calculate the stats for each cell independently + while ((cellIn = pmFPAviewNextCell(view, fpaIn, 1))) { + if (cellIn->readouts->n == 0) continue; + pmCell *cellOut = pmFPAviewThisCell(view, fpaOut); // Output cell + + pmReadout *roOut = NULL; + if (cellOut->readouts->n > 0) { + roOut = cellOut->readouts->data[0]; + psTrace("ppMerge", 5, "masking with results from previous pass\n"); + } + + pmReadout *roIn; // Input readout of interest + while ((roIn = pmFPAviewNextReadout(view, fpaIn, 1))) { + if (!roIn->image) continue; + psTrace("ppMerge", 4, "Measure statistics for chip %d, cell %d, ro %d\n", + view->chip, view->cell, view->readout); + ppMergeMaskReadoutStats (roIn, roOut, options, rng); + } + } + } + + // apply the measured statistics to determine the outliers to be masked + while ((cellIn = pmFPAviewNextCell(view, fpaIn, 1))) { + if (cellIn->readouts->n == 0) continue; + + pmCell *cellOut = pmFPAviewThisCell(view, fpaOut); // Output cell + // Suspect pixels image + psImage *suspect = psMetadataLookupPtr(NULL, cellOut->analysis, "MASK.SUSPECT"); + bool first = suspect ? false : true; + + pmReadout *roIn; // Input readout of interest + while ((roIn = pmFPAviewNextReadout(view, fpaIn, 1))) { + if (!roIn->image) { + continue; + } + psTrace("ppMerge", 4, "Flagging suspect pixels in chip %d, cell %d, ro %d\n", + view->chip, view->cell, view->readout); + suspect = pmMaskFlagSuspectPixels(suspect, roIn, options->maskSuspect, options->combine->maskVal); + } + + if (first) { + psMetadataAddImage(cellOut->analysis, PS_LIST_TAIL, "MASK.SUSPECT", 0, + "Suspect pixels", suspect); + psFree(suspect); + } + pmCellFreeData(cellIn); + } + pmChipFreeData(chipIn); + } + pmFPAFreeData(fpaIn); + psFree(view); + } + psFree(rng); + + return true; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeMaskWrite.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeMaskWrite.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeMaskWrite.c (revision 17232) @@ -0,0 +1,69 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeData.h" +#include "ppMergeMask.h" + + +// Generate a mask +bool ppMergeMaskWrite(ppMergeData *data, // Data + ppMergeOptions *options, // Options + pmConfig *config // Configuration + ) +{ + pmFPA *fpaOut = data->out; // Output FPA + + pmFPAview *view = pmFPAviewAlloc(0);// View of FPA, for iteration + if (fpaOut->hdu) { + pmFPAUpdateNames(fpaOut, NULL, NULL); + } + pmFPAWriteMask(fpaOut, data->outFile, config->database, true, false); // Write header only + pmChip *chipOut; // Output chip of interest + while ((chipOut = pmFPAviewNextChip(view, fpaOut, 1))) { + if (chipOut->hdu) { + chipOut->data_exists = true; + pmFPAUpdateNames(fpaOut, chipOut, NULL); + } + pmChipWriteMask(chipOut, data->outFile, config->database, true, false); // Write header only + pmCell *cellOut; // Output cell of interest + while ((cellOut = pmFPAviewNextCell(view, fpaOut, 1))) { + if (cellOut->hdu) { + chipOut->data_exists = cellOut->data_exists = true; + pmFPAUpdateNames(fpaOut, chipOut, cellOut); + } + pmCellWriteMask(cellOut, data->outFile, config->database, true); // Write header only + + if (cellOut->hdu && !cellOut->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out cell HDU.\n"); + pmCellWriteMask(cellOut, data->outFile, config->database, false); + pmCellFreeData(cellOut); + } + } + + if (chipOut->hdu && !chipOut->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out chip HDU.\n"); + pmChipWriteMask(chipOut, data->outFile, config->database, false, false); + pmChipFreeData(chipOut); + } + } + + if (fpaOut->hdu && !fpaOut->hdu->blankPHU) { + psTrace("ppMerge", 5, "Writing out FPA HDU.\n"); + pmFPAWriteMask(fpaOut, data->outFile, config->database, false, false); + } + pmFPAFreeData(fpaOut); + + psFree(view); + + return true; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeOptions.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeOptions.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeOptions.c (revision 17232) @@ -0,0 +1,323 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "ppMerge.h" +#include "ppMergeOptions.h" + +#define ARRAY_BUFFER 4 // Number of values to add at once + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ppMergeOptions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Free function +static void mergeOptionsFree(ppMergeOptions *options // Options to free + ) +{ + psFree(options->format); + psFree(options->combine); + psFree(options->darkOrdinates); + psFree(options->darkNorm); +} + +// Allocator +ppMergeOptions *ppMergeOptionsAlloc(const pmConfig *config) +{ + ppMergeOptions *options = psAlloc(sizeof(ppMergeOptions)); // The options, to return + psMemSetDeallocator(options, (psFreeFunc)mergeOptionsFree); + + options->format = NULL; + options->rows = 0; + options->minElectrons = NAN; + options->zero = false; + options->scale = false; + options->dark = false; + options->fringe = false; + options->shutter = false; + options->mask = false; + options->sample = 1; + options->mean = PS_STAT_SAMPLE_MEDIAN; + options->stdev = PS_STAT_SAMPLE_STDEV; + options->fringeNum = 100; + options->fringeSize = 10; + options->fringeSmoothX = 5; + options->fringeSmoothY = 5; + options->shutterSize = 10; + options->shutterIter = 2; + options->shutterRej = 3.0; + options->maskSuspect = 5.0; + options->maskBad = 10.0; + options->maskMode = PM_MASK_ID_VALUE; + options->statsByChip = true; + options->onOff = 0; + options->combine = pmCombineParamsAlloc(PS_STAT_SAMPLE_MEAN); + options->combine->rej = 3.0; + options->combine->iter = 1; + options->combine->fracHigh = 0.0; + options->combine->fracLow = 0.0; + options->combine->nKeep = 1; + options->combine->maskVal = 0xff; + options->combine->weights = false; + options->combine->blank = pmConfigMask("BLANK", config); + options->satMask = 0x00; + options->badMask = 0x00; + options->growPixels = 0; + options->darkOrdinates = NULL; + options->darkNorm = NULL; + return options; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ppMergeOptionsParse +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Parse a recipe option according to its type +#define OPTION_PARSE(OPTION,MD,NAME,TYPE) \ +{ \ + psMetadataItem *item = psMetadataLookup(MD, NAME); \ + if (item) { \ + OPTION = psMetadataItemParse##TYPE(item); \ + } else { \ + psWarning("Recipe option %s isn't specified; using default.\n", NAME); \ + } \ +} + +// Parse a statistic +static psStatsOptions parseStat(psMetadata *source, // Source of the statistics option + const char *name // Name of the statistics option + ) +{ + bool mdok = true; // Status of MD lookup + const char *stat = psMetadataLookupStr(&mdok, source, name); // The statistic string + if (!mdok || !stat || strlen(stat) == 0) { + return 0; + } + return psStatsOptionFromString(stat); +} + +// Parse the options +ppMergeOptions *ppMergeOptionsParse(pmConfig *config // Configuration + ) +{ + ppMergeOptions *options = ppMergeOptionsAlloc(config); // The merge options + + // We need to work out the camera before we can get the recipe. Take the first input and inspect it. + if (!config->camera) { + psArray *filenames = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); // The input file names + psString resolved = pmConfigConvertFilename(filenames->data[0], config, false); // Resolved file name + psFits *inFile = psFitsOpen(resolved, "r"); // The FITS file to read + if (!inFile) { + psError(PS_ERR_IO, false, "Unable to open input file %s to determine camera.\n", resolved); + psFree(resolved); + psFree(config); + exit(EXIT_FAILURE); + } + psFree(resolved); + psMetadata *header = psFitsReadHeader(NULL, inFile); // The FITS (primary) header + psFitsClose(inFile); + + options->format = pmConfigCameraFormatFromHeader(config, header, true); + psFree(header); + if (!options->format) { + psLogMsg(__func__, PS_LOG_WARN, "Unable to identify camera format for input file %s\n", + (char *)filenames->data[0]); + exit(EXIT_FAILURE); + } + } + + // we must have the recipes by this point + assert (config->recipes); + + // Now we can read the recipe + bool mdok = true; // Status of MD lookup + psMetadata *recipe = psMetadataLookupMetadata(&mdok, config->recipes, PPMERGE_RECIPE); // Recipe information + if (!mdok || !recipe) { + psError(PS_ERR_IO, true, "Unable to find recipe %s", PPMERGE_RECIPE); + exit(EXIT_FAILURE); + } + + // First, deal with the recipe. These are parameters that will typically be constant for a camera. + OPTION_PARSE(options->rows, recipe, "ROWS", S32); + OPTION_PARSE(options->minElectrons, recipe, "ELECTRONS", F32); + OPTION_PARSE(options->sample, recipe, "SAMPLE", S32); + OPTION_PARSE(options->combine->rej, recipe, "REJ", F32); + OPTION_PARSE(options->combine->iter, recipe, "ITER", S32); + OPTION_PARSE(options->combine->fracHigh, recipe, "FRACHIGH", F32); + OPTION_PARSE(options->combine->fracLow, recipe, "FRACLOW", F32); + OPTION_PARSE(options->combine->nKeep, recipe, "NKEEP", S32); + OPTION_PARSE(options->combine->weights, recipe, "WEIGHTS", Bool); + OPTION_PARSE(options->fringeNum, recipe, "FRINGE.NUM", S32); + OPTION_PARSE(options->fringeSize, recipe, "FRINGE.SIZE", S32); + OPTION_PARSE(options->fringeSmoothX, recipe, "FRINGE.XSMOOTH", S32); + OPTION_PARSE(options->fringeSmoothY, recipe, "FRINGE.YSMOOTH", S32); + OPTION_PARSE(options->shutterSize, recipe, "SHUTTER.SIZE", S32); + OPTION_PARSE(options->shutterIter, recipe, "SHUTTER.ITER", S32); + OPTION_PARSE(options->shutterRej, recipe, "SHUTTER.REJECT", F32); + OPTION_PARSE(options->maskSuspect, recipe, "MASK.SUSPECT", F32); + OPTION_PARSE(options->maskBad, recipe, "MASK.BAD", F32); + OPTION_PARSE(options->statsByChip, recipe, "STATS.BY.CHIP", Bool); + OPTION_PARSE(options->growPixels, recipe, "MASK.GROW.NPIX", S32); + + const char *masks = psMetadataLookupStr(NULL, recipe, "MASKVAL"); + options->combine->maskVal = pmConfigMask(masks, config); + + options->combine->combine = parseStat(recipe, "COMBINE"); + options->mean = parseStat(recipe, "MEAN"); + options->stdev = parseStat(recipe, "STDEV"); + + // Now the command-line options. These are parameters that depend on what type of frame is being combined + + // Set options based on the type of calibration frame + const char *type = psMetadataLookupStr(NULL, config->arguments, "-type"); // The type of calibration frame + if (strlen(type) > 0) { + if (strcasecmp(type, "BIAS") == 0) { + options->zero = false; + options->scale = false; + options->dark = false; + options->fringe = false; + options->shutter = false; + options->mask = false; + } else if (strcasecmp(type, "DARK") == 0) { + options->zero = false; + options->scale = false; + options->dark = true; + options->fringe = false; + options->shutter = false; + options->mask = false; + } else if (strcasecmp(type, "FLAT") == 0) { + options->zero = false; + options->scale = true; + options->dark = false; + options->fringe = false; + options->shutter = false; + options->mask = false; + } else if (strcasecmp(type, "SKYFLAT") == 0) { + options->zero = false; + options->scale = true; + options->dark = false; + options->fringe = false; + options->shutter = false; + options->mask = false; + } else if (strcasecmp(type, "DOMEFLAT") == 0) { + options->zero = false; + options->scale = true; + options->dark = false; + options->fringe = false; + options->shutter = false; + options->mask = false; + } else if (strcasecmp(type, "FRINGE") == 0) { + options->zero = true; + options->scale = true; + options->dark = false; + options->fringe = true; + options->shutter = false; + options->mask = false; + } else if (strcasecmp(type, "SHUTTER") == 0) { + options->zero = false; + options->scale = false; + options->dark = false; + options->fringe = false; + options->shutter = true; + options->mask = false; + } else if (strcasecmp(type, "MASK") == 0 || + strcasecmp(type, "DARKMASK") == 0 || + strcasecmp(type, "FLATMASK") == 0) { + options->zero = false; + options->scale = false; + options->dark = false; + options->fringe = false; + options->shutter = false; + options->mask = true; + } else { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unrecognised image type: %s", type); + psFree(options); + return NULL; + } + } else { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "No image type specified."); + psFree(options); + return NULL; + } + + const char *maskMode = psMetadataLookupStr(NULL, recipe, "MASK.MODE"); // The type of calibration frame + if (!maskMode) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "No mask mode specified in recipe."); + psFree(options); + return NULL; + } + + options->maskMode = pmMaskIdentifyModeFromString (maskMode); + if (options->maskMode == PM_MASK_ID_NONE) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "invalid mask mode %s.", maskMode); + psFree(options); + return NULL; + } + +#if 0 + // Number of on/off images + OPTION_PARSE(options->onOff, config->arguments, "-onoff", S32); +#endif + + // Masking options + options->satMask = pmConfigMask("SAT", config); + options->badMask = pmConfigMask("BAD", config); + + if (options->dark) { + psMetadata *ordinates = psMetadataLookupMetadata(NULL, recipe, "DARK.ORDINATES"); // Ordinates info + options->darkOrdinates = psArrayAllocEmpty(psListLength(ordinates->list)); + + psMetadataIterator *iter = psMetadataIteratorAlloc(ordinates, PS_LIST_HEAD, NULL); // Iterator + psMetadataItem *item; // Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + int order = 0; // Polynomial order + bool scale = false; // Scale values? + float min = NAN, max = NAN; // Minimum and maximum values for scaling + switch (item->type) { + case PS_TYPE_S32: + order = item->data.S32; + break; + case PS_DATA_METADATA: + order = psMetadataLookupS32(NULL, item->data.md, "ORDER"); + bool mdok; // Status of MD lookup + scale = psMetadataLookupBool(&mdok, item->data.md, "SCALE"); + min = psMetadataLookupF32(&mdok, item->data.md, "MIN"); + max = psMetadataLookupF32(&mdok, item->data.md, "MAX"); + break; + default: + psError(PS_ERR_BAD_PARAMETER_TYPE, true, + "Type of DARK.ORDINATES entry %s (%x) is not METADATA or S32", + item->name, item->type); + psFree(options); + return false; + } + if (order <= 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "ORDER not positive (%d) for DARK.ORDINATES %s", + order, item->name); + psFree(options); + return false; + } + + pmDarkOrdinate *ord = pmDarkOrdinateAlloc(item->name, order); + ord->scale = scale; + ord->min = min; + ord->max = max; + psArrayAdd(options->darkOrdinates, options->darkOrdinates->n, ord); + psFree(ord); + } + psFree(iter); + + psString darkNorm = psMetadataLookupStr(NULL, recipe, "DARK.NORM"); // Normalisation concept + if (darkNorm && strcmp(darkNorm, "NONE") != 0) { + options->darkNorm = psStringCopy(darkNorm); + } + } + + return options; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeOptions.h =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeOptions.h (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeOptions.h (revision 17232) @@ -0,0 +1,59 @@ +#ifndef PP_MERGE_OPTIONS_H +#define PP_MERGE_OPTIONS_H + +#include +#include + +// Mode of on/off pairs; the value corresponds to how many images are in each set +typedef enum { + PP_ONOFF_ABBA = -1, // On/off pairs in the ABBA mode + PP_ONOFF_NONE = 0, // No on/off pairs + PP_ONOFF_ABAB = 1, // On/off pairs in the ABAB mode (one image each) + PP_ONOFF_AABB = 2, // On/off pairs, two images each + // And so on and so forth... just use a number beyond this +} ppOnOff; + +// Options for ppMerge +typedef struct { + psMetadata *format; // Camera format configuration + unsigned int rows; // Number of rows to read at once + float minElectrons; // Minimum number of electrons for useful signal + bool zero; // Subtract background before combining? + bool scale; // Scale by the background before combining? + bool dark; // Generate dark? + bool fringe; // Make fringe measurements? + bool shutter; // Generate shutter correction? + bool mask; // Generate bad pixel mask? + unsigned int sample; // Sampling factor for measuring the background + psStatsOptions mean; // Statistic to use to measure the mean + psStatsOptions stdev; // Statistic to use to measure the stdev + int fringeNum; // Number of fringe regions per cell + int fringeSize; // Size of fringe regions + int fringeSmoothX; // Number of smoothing regions per cell, in x + int fringeSmoothY; // Number of smoothing regions per cell, in y + int shutterSize; // Size for shutter measurement regions + int shutterIter; // Number of iterations for shutter measurement + float shutterRej; // Rejection limit for shutter measurement + float maskSuspect; // Threshold for identifying suspect pixels + float maskBad; // Threshold for identifying bad pixels + bool statsByChip; // measure statistics for masking by chip or readout? + pmMaskIdentifyMode maskMode; // how to set the limit based on the threshold value above? + ppOnOff onOff; // On/off pairs? + pmCombineParams *combine; // Combination parameters + psMaskType satMask; // Mask value for saturated pixels + psMaskType badMask; // Mask value for bad (low) pixels + int growPixels; + psArray *darkOrdinates; // Ordinates for dark combination + psString darkNorm; // Dark normalisation concept +} ppMergeOptions; + +// Allocator +ppMergeOptions *ppMergeOptionsAlloc(const pmConfig *config); + + +// Parse the options for ppMerge +ppMergeOptions *ppMergeOptionsParse(pmConfig *config // Configuration + ); + + +#endif Index: /tags/pap_merge_030828/ppMerge/src/ppMergeScaleZero.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeScaleZero.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeScaleZero.c (revision 17232) @@ -0,0 +1,216 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "ppMerge.h" + +// Get the scale and zero for each chip of each input +bool ppMergeScaleZero(pmConfig *config) +{ + assert(config); + + ppMergeType type = psMetadataLookupS32(NULL, config->arguments, "TYPE"); // Type of frame + int numInputs = psMetadataLookupS32(NULL, config->arguments, "INPUTS.NUM"); // Number of inputs + int numCells = psMetadataLookupS32(NULL, config->arguments, "INPUTS.CELLS"); // Number of cells + psStatsOptions meanStat = psMetadataLookupS32(NULL, config->arguments, "MEAN"); // Statistic for mean + psStatsOptions stdevStat = psMetadataLookupS32(NULL, config->arguments, "STDEV"); // Statistic for stdev + psMaskType maskVal = psMetadataLookupU8(NULL, config->arguments, "MASKVAL"); // Value to mask + int shutterSize = psMetadataLookupS32(NULL, config->arguments, "SHUTTER.SIZE"); // Size of shutter region + + psVector *gains = NULL; // Gains for each cell + psArray *shutters = NULL; // Shutter data for each cell + psStats *stats = NULL; // Statistics for background + psImage *background = NULL; // Background measurements per cell per file + + switch (type) { + case PPMERGE_TYPE_BIAS: + case PPMERGE_TYPE_DARK: + // Nothing to measure + return true; + case PPMERGE_TYPE_FLAT: + case PPMERGE_TYPE_FRINGE: + gains = psVectorAlloc(numCells, PS_TYPE_F32); + background = psImageAlloc(numCells, numInputs, PS_TYPE_F32); + psImageInit(background, NAN); + stats = psStatsAlloc(meanStat); + break; + case PPMERGE_TYPE_SHUTTER: + shutters = psArrayAlloc(numCells); + break; + case PPMERGE_TYPE_MASK: + default: + break; + } + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator + pmFPAview *view = NULL; // View into FPA + + for (int i = 0; i < numInputs; i++) { + pmFPAfileActivate(config->files, false, NULL); + psArray *files = ppMergeFileActivateSingle(config, PPMERGE_FILES_INPUT, true, i); // Activated files + pmFPAfile *input = files->data[0]; // Representative file; should be the image (not mask or weight) + pmFPA *fpa = input->fpa; // FPA of interest + view = pmFPAviewAlloc(0); // View to component of interest + int cellNum = 0; // Index for cell + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + goto ERROR; + } + pmChip *chip; // Chip of interest + while ((chip = pmFPAviewNextChip(view, fpa, 1))) { + if (!chip->file_exists) { + continue; + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + goto ERROR; + } + + pmCell *cell; // Cell of interest + while ((cell = pmFPAviewNextCell(view, fpa, 1))) { + if (!cell->file_exists) { + continue; + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + goto ERROR; + } + + if (!cell->data_exists) { + continue; + } + + if (cell->readouts->n > 1) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "File %d chip %d cell %d contains more than one readout (%ld)", + i, view->chip, view->cell, cell->readouts->n); + goto ERROR; + } + pmReadout *readout = cell->readouts->data[0]; // Readout of interest + + switch (type) { + case PPMERGE_TYPE_FLAT: + case PPMERGE_TYPE_FRINGE: { + // Extract the gain + float gain = psMetadataLookupF32(NULL, cell->concepts, "CELL.GAIN"); // Cell gain + if (!isfinite(gain)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, false, + "CELL.GAIN for file %d chip %d cell %d is not set.", + i, view->chip, view->cell); + goto ERROR; + } + gains->data.F32[cellNum] = gain; + + // Measure the background + if (!psImageBackground(stats, NULL, readout->image, readout->mask, maskVal, rng)) { + psError(PS_ERR_UNKNOWN, false, + "Unable to get statistics for file %d chip %d cell %d", + i, view->chip, view->cell); + goto ERROR; + } + background->data.F32[i][cellNum] = psStatsGetValue(stats, meanStat); + break; + } + case PPMERGE_TYPE_SHUTTER: { + pmShutterCorrectionData *shutter = shutters->data[cellNum]; // Shutter correction data + if (!shutter) { + shutter = pmShutterCorrectionDataAlloc(readout->image->numCols, + readout->image->numRows, + shutterSize); + shutters->data[cellNum] = shutter; + } + if (!pmShutterCorrectionAddReadout(shutter, readout, meanStat, stdevStat, + maskVal, rng)) { + psError(PS_ERR_UNKNOWN, false, + "Can't add file %d chip %d cell %d to shutter correction.", + i, view->chip, view->cell); + goto ERROR; + } + break; + } + case PPMERGE_TYPE_MASK: + default: + psAbort("Should never get here."); + } + + cellNum++; + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + goto ERROR; + } + } + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + goto ERROR; + } + } + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + goto ERROR; + } + + psFree(view); + +#if 0 + // Reset files for reading again + for (int i = 0; i < files->n; i++) { + pmFPAfile *file = files->data[i]; // File of interest + } +#endif + psFree(files); + } + + psFree(rng); rng = NULL; + psFree(stats); stats = NULL; + + // Store results + switch (type) { + case PPMERGE_TYPE_FRINGE: + psMetadataAddImage(config->arguments, PS_LIST_TAIL, "ZEROS", 0, + "Zero to subtract from each input cell", background); + // Flow through + case PPMERGE_TYPE_FLAT: { + // Need to normalize over the focal plane + if (psTraceGetLevel("ppMerge") > 9) { + for (int i = 0; i < gains->n; i++) { + psTrace("ppMerge", 10, "Gain for cell %d is %f\n", i, gains->data.F32[i]); + } + } + psVector *fluxes = NULL; // Solution to fluxes + if (!pmFlatNormalize(&fluxes, &gains, background)) { + psError(PS_ERR_UNKNOWN, false, "Normalisation failed to converge --- continuing anyway."); + psFree(fluxes); + goto ERROR; + } + + psMetadataAddVector(config->arguments, PS_LIST_TAIL, "SCALES", 0, + "Scale to divide into each input file", fluxes); + psFree(fluxes); // Drop reference + break; + } + case PPMERGE_TYPE_SHUTTER: + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "SHUTTER", 0, + "Shutter data", shutters); + break; + case PPMERGE_TYPE_MASK: + default: + psAbort("Should never get here."); + } + + psFree(background); + psFree(shutters); + return true; + +ERROR: + // Common path for errors + psFree(gains); + psFree(background); + psFree(shutters); + psFree(rng); + psFree(stats); + psFree(view); + return false; +} + Index: /tags/pap_merge_030828/ppMerge/src/ppMergeScaleZero.h =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeScaleZero.h (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeScaleZero.h (revision 17232) @@ -0,0 +1,19 @@ +#ifndef PP_MERGE_SCALE_ZERO_H +#define PP_MERGE_SCALE_ZERO_H + +#include +#include + +#include "ppMergeData.h" +#include "ppMergeOptions.h" + +// Get the scale and zero for each chip of each input +bool ppMergeScaleZero(psImage **scales, // The scales for each integration/cell + psImage **zeros, // The zeroes for each integration/cell + psArray **shutter,// The shutter correction data for each cell + ppMergeData *data,// The data + const ppMergeOptions *options, // The options + const pmConfig *config // The configuration + ); + +#endif Index: /tags/pap_merge_030828/ppMerge/src/ppMergeVersion.c =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeVersion.c (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeVersion.c (revision 17232) @@ -0,0 +1,60 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppMergeVersion.h" + +static const char *cvsTag = "$Name: not supported by cvs2svn $";// CVS tag name + +psString ppMergeVersion(void) +{ + psString version = NULL; // Version, to return + psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION); + return version; +} + +psString ppMergeVersionLong(void) +{ + psString version = ppMergeVersion(); // Version, to return + psString tag = psStringStripCVS(cvsTag, "Name"); // CVS tag + psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__); + psFree(tag); + return version; +} + + +void ppMergeVersionMetadata(psMetadata *metadata) +{ + PS_ASSERT_METADATA_NON_NULL(metadata,); + + psString pslib = psLibVersionLong();// psLib version + psString psmodules = psModulesVersionLong(); // psModules version + psString ppStats = ppStatsVersionLong(); // ppStats version + psString ppMerge = ppMergeVersionLong(); // ppMerge version + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psString head = NULL; // Head string + psStringAppend(&head, "ppMerge processing at %s. Component information:", timeString); + psFree(timeString); + + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, head, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, pslib, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, psmodules, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppStats, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppMerge, ""); + + psFree(head); + psFree(pslib); + psFree(psmodules); + psFree(ppStats); + psFree(ppMerge); + + return; +} Index: /tags/pap_merge_030828/ppMerge/src/ppMergeVersion.h =================================================================== --- /tags/pap_merge_030828/ppMerge/src/ppMergeVersion.h (revision 17232) +++ /tags/pap_merge_030828/ppMerge/src/ppMergeVersion.h (revision 17232) @@ -0,0 +1,14 @@ +#ifndef PP_MERGE_VERSION_H +#define PP_MERGE_VERSION_H + +/// Return short version information +psString ppMergeVersion(void); + +/// Return long version information +psString ppMergeVersionLong(void); + +/// Update the metadata with version information for all dependencies +void ppMergeVersionMetadata(psMetadata *metadata ///< Metadata to update with version information + ); + +#endif Index: /tags/pap_merge_030828/psModules/src/camera/pmReadoutStack.c =================================================================== --- /tags/pap_merge_030828/psModules/src/camera/pmReadoutStack.c (revision 17232) +++ /tags/pap_merge_030828/psModules/src/camera/pmReadoutStack.c (revision 17232) @@ -0,0 +1,184 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pmReadoutStack.h" + +psImage *pmReadoutAnalysisImage(pmReadout *readout, // Readout containing image + const char *name, // Name of image in analysis metadata + int numCols, int numRows, // Expected size of image + psElemType type, // Expected type of image + double init // Initial value + ) +{ + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_STRING_NON_EMPTY(name, false); + + bool mdok; // Status of MD lookup + psImage *image = psMetadataLookupPtr(&mdok, readout->analysis, name); + if (!image) { + image = psImageAlloc(numCols, numRows, type); + psMetadataAddImage(readout->analysis, PS_LIST_TAIL, name, 0, "Analysis image from " __FILE__, image); + psImageInit(image, init); + return image; + } + if (image->numCols != numCols || image->numRows != numRows) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, "Analysis image %s has incorrect size (%dx%d vs %dx%d)", + name, image->numCols, image->numRows, numCols, numRows); + return NULL; + } + if (image->type.type != type) { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Analysis image %s has incorrect type (%x vs %x)", + name, image->type.type, type); + return NULL; + } + return psMemIncrRefCounter(image); +} + +bool pmReadoutUpdateSize(pmReadout *readout, int minCols, int minRows, + int numCols, int numRows, bool mask, bool weight, + psMaskType blank) +{ + PS_ASSERT_PTR_NON_NULL(readout, false); + + if (readout->image) { + readout->col0 = PS_MIN(minCols, readout->col0); + readout->row0 = PS_MIN(minRows, readout->row0); + } else { + readout->col0 = minCols; + readout->row0 = minRows; + } + + // (reAllocate the images + if (!readout->image) { + readout->image = psImageAlloc(numCols, numRows, PS_TYPE_F32); + psImageInit(readout->image, NAN); + } + if (readout->image->numCols < numCols || readout->image->numRows < numRows) { + // Generate the new output image by extending the current one, or making a whole new one + psImage *newImage = psImageAlloc(numCols, numRows, PS_TYPE_F32); + psImageInit(newImage, NAN); + psImageOverlaySection(newImage, readout->image, readout->col0, readout->row0, "="); + psFree(readout->image); + readout->image = newImage; + } + + if (mask) { + if (!readout->mask) { + readout->mask = psImageAlloc(numCols, numRows, PS_TYPE_MASK); + psImageInit(readout->mask, blank); + } + if (readout->mask->numCols < numCols || readout->mask->numRows < numRows) { + psImage *newMask = psImageAlloc(numCols, numRows, PS_TYPE_MASK); + psImageInit(newMask, blank); + psImageOverlaySection(newMask, readout->mask, readout->col0, readout->row0, "="); + psFree(readout->mask); + readout->mask = newMask; + } + } + + if (weight) { + if (!readout->weight) { + readout->weight = psImageAlloc(numCols, numRows, PS_TYPE_F32); + psImageInit(readout->weight, NAN); + } + if (readout->weight->numCols < numCols || readout->weight->numRows < numRows) { + psImage *newWeight = psImageAlloc(numCols, numRows, PS_TYPE_F32); + psImageInit(newWeight, NAN); + psImageOverlaySection(newWeight, readout->weight, readout->col0, readout->row0, "="); + psFree(readout->weight); + readout->weight = newWeight; + } + } + + return true; +} + +bool pmReadoutStackValidate(int *minInputColsPtr, int *maxInputColsPtr, int *minInputRowsPtr, + int *maxInputRowsPtr, int *numColsPtr, int *numRowsPtr, + const psArray *inputs) +{ + PS_ASSERT_ARRAY_NON_NULL(inputs, false); + PS_ASSERT_PTR_NON_NULL(minInputColsPtr, false); + PS_ASSERT_PTR_NON_NULL(maxInputColsPtr, false); + PS_ASSERT_PTR_NON_NULL(minInputRowsPtr, false); + PS_ASSERT_PTR_NON_NULL(maxInputRowsPtr, false); + PS_ASSERT_PTR_NON_NULL(numColsPtr, false); + PS_ASSERT_PTR_NON_NULL(numRowsPtr, false); + + // Step through each readout in the input image list to determine how big of an output image is needed to + // combine these input images. + int maxInputCols = 0; // The largest input column value + int maxInputRows = 0; // The largest input row value + int minInputCols = INT_MAX; // The smallest input column value + int minInputRows = INT_MAX; // The smallest input row value + int xSize = 0, ySize = 0; // The size of the output image + + int xMin = INT_MAX; + int yMin = INT_MAX; + int xMax = 0; + int yMax = 0; + + bool valid = false; // Do we have a single valid input? + for (long i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + + if (!readout) { + continue; + } + if (!readout->image) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Input readout %ld has NULL image.\n", i); + return false; + } + + // use the trimsec to define the max full range of the output pixels + pmCell *cell = readout->parent; // The parent cell + bool mdok = true; // Status of MD lookup + psRegion *trimsec = psMetadataLookupPtr(&mdok, cell->concepts, "CELL.TRIMSEC"); // Trim section + if (!mdok || !trimsec || psRegionIsNaN(*trimsec)) { + psWarning("CELL.TRIMSEC is not set for readout %ld --- ignored.\n", i); + } else { + xSize = PS_MAX(xSize, trimsec->x1 - trimsec->x0); + ySize = PS_MAX(ySize, trimsec->y1 - trimsec->y0); + xMin = PS_MIN(xMin, trimsec->x0); + xMax = PS_MAX(xMax, trimsec->x1); + yMin = PS_MIN(yMin, trimsec->y0); + yMax = PS_MAX(yMax, trimsec->y1); + } + + valid = true; + + // Range of pixels on output images + minInputCols = PS_MAX(xMin, PS_MIN(minInputCols, readout->col0)); + maxInputCols = PS_MIN(xMax, PS_MAX(maxInputCols, readout->col0 + readout->image->numCols)); + minInputRows = PS_MAX(yMin, PS_MIN(minInputRows, readout->row0)); + maxInputRows = PS_MIN(yMax, PS_MAX(maxInputRows, readout->row0 + readout->image->numRows)); + psTrace("psModules.camera", 7, "Readout %ld: offset %d,%d; size %dx%d\n", i, + readout->col0, readout->row0, readout->image->numCols, readout->image->numRows); + } + + if (minInputColsPtr) { + *minInputColsPtr = minInputCols; + } + if (maxInputColsPtr) { + *maxInputColsPtr = maxInputCols; + } + if (minInputRowsPtr) { + *minInputRowsPtr = minInputRows; + } + if (maxInputRowsPtr) { + *maxInputRowsPtr = maxInputRows; + } + if (numColsPtr) { + *numColsPtr = xSize; + } + if (numRowsPtr) { + *numRowsPtr = ySize; + } + + return valid; +} Index: /tags/pap_merge_030828/psModules/src/camera/pmReadoutStack.h =================================================================== --- /tags/pap_merge_030828/psModules/src/camera/pmReadoutStack.h (revision 17232) +++ /tags/pap_merge_030828/psModules/src/camera/pmReadoutStack.h (revision 17232) @@ -0,0 +1,34 @@ +#ifndef PM_READOUT_STACK_H +#define PM_READOUT_STACK_H + +#include "pmHDU.h" +#include "pmFPA.h" + +#define PM_READOUT_STACK_ANALYSIS_COUNT "STACK.COUNT" // Name for count image in analysis metadata +#define PM_READOUT_STACK_ANALYSIS_SIGMA "STACK.SIGMA" // Name for sigma image in analysis metadata + +/// Update an output readout (for a stack) with the correct col0,row0 and the image size +bool pmReadoutUpdateSize(pmReadout *readout, ///< Readout which to update + int minCols, int minRows, ///< Minimum coordinates + int numCols, int numRows, ///< Size of images + bool mask, ///< Worry about the mask? + bool weight, ///< Worry about the weight? + psMaskType blank ///< Mask value to give to blank pixels + ); + +/// Determine how large an output image is needed to combine the input readouts +bool pmReadoutStackValidate(int *minInputColsPtr, int *maxInputColsPtr, ///< Min and max size in x + int *minInputRowsPtr, int *maxInputRowsPtr, ///< Min and max size in y + int *numColsPtr, int *numRowsPtr, ///< Size of image + const psArray *inputs ///< Array of pmReadouts + ); + +/// Return an image from analysis metadata, produced while stacking +psImage *pmReadoutAnalysisImage(pmReadout *readout, // Readout containing image + const char *name, // Name of image in analysis metadata + int numCols, int numRows, // Expected size of image + psElemType type, // Expected type of image + double init // Initial value + ); + +#endif Index: /tags/pap_merge_030828/psModules/src/detrend/pmDark.c =================================================================== --- /tags/pap_merge_030828/psModules/src/detrend/pmDark.c (revision 17232) +++ /tags/pap_merge_030828/psModules/src/detrend/pmDark.c (revision 17232) @@ -0,0 +1,759 @@ +#include +#include +#include + +#include "psPolynomialMD.h" + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmHDUUtils.h" +#include "pmFPARead.h" +#include "pmFPAWrite.h" +#include "pmReadoutStack.h" + +#include "pmDark.h" + +#define PM_DARK_FITS_EXTNAME "PS_DARK" // FITS extension name for ordinates table +#define PM_DARK_FITS_NAME "NAME" // Column name for concept name in ordinates table +#define PM_DARK_FITS_ORDER "ORDER" // Column name for polynomial order in ordinates table +#define PM_DARK_FITS_SCALE "SCALE" // Column name for scaling option in ordinates table +#define PM_DARK_FITS_MIN "MIN" // Column name for minimum value in ordinates table +#define PM_DARK_FITS_MAX "MAX" // Column name for maximum value in ordinates table + + + +// Look up the value of an ordinate in a readout +static bool ordinateLookup(float *value, // Value of ordinate, to return + const char *name, // Name of ordinate (concept name) + bool scale, // Scale the value? + float min, float max, // Minimum and maximum values for scaling + const pmReadout *ro // Readout of interest + ) +{ + assert(value); + assert(name); + assert(ro); + + pmCell *cell = ro->parent; // Parent cell + if (!cell) { + return false; + } + psMetadataItem *item = psMetadataLookup(cell->concepts, name); + if (!item) { + pmChip *chip = cell->parent; // Parent chip + if (!chip) { + return false; + } + item = psMetadataLookup(chip->concepts, name); + if (!item) { + pmFPA *fpa = chip->parent; // Parent FPA + if (!fpa) { + return false; + } + item = psMetadataLookup(fpa->concepts, name); + if (!item) { + psWarning("Unable to find concept %s in readout", name); + return false; + } + } + } + + *value = psMetadataItemParseF32(item); // Value of interest + if (!isfinite(*value)) { + psWarning("Non-finite value (%f) of concept %s in readout", *value, name); + return false; + } + if (scale) { + if (*value < min || *value > max) { + psWarning("Value of concept %s (%f) outside range (%f:%f)", name, *value, min, max); + return false; + } + *value = 2.0 * (*value - min) / (max - min) - 1.0; + } + + return true; +} + +static void darkOrdinateFree(pmDarkOrdinate *ord) +{ + psFree(ord->name); + return; +} + +pmDarkOrdinate *pmDarkOrdinateAlloc(const char *name, int order) +{ + pmDarkOrdinate *ord = psAlloc(sizeof(pmDarkOrdinate)); // Ordinate data, to return + psMemSetDeallocator(ord, (psFreeFunc)darkOrdinateFree); + + ord->name = psStringCopy(name); + ord->order = order; + ord->scale = false; + ord->min = NAN; + ord->max = NAN; + + return ord; +} + + +bool pmDarkCombine(pmCell *output, const psArray *inputs, psArray *ordinates, const char *normConcept, + int iter, float rej, psMaskType maskVal) +{ + PS_ASSERT_PTR_NON_NULL(output, false); + PS_ASSERT_ARRAY_NON_NULL(inputs, false); + PS_ASSERT_ARRAY_NON_NULL(ordinates, false); + PS_ASSERT_INT_NONNEGATIVE(iter, false); + PS_ASSERT_FLOAT_LARGER_THAN(rej, 0.0, false); + + // Extract fitting orders + int numOrdinates = ordinates->n; // Number of ordinates + int numInputs = inputs->n; // Number of inputs + int numBadInputs = 0; // Number of bad inputs + psArray *values = psArrayAlloc(numInputs); + psVector *roMask = psVectorAlloc(numInputs, PS_TYPE_U8); // Mask for bad readouts + psVectorInit(roMask, 0); + psVector *norm = normConcept ? psVectorAlloc(numInputs, PS_TYPE_F32) : NULL; // Normalisations for each + for (int i = 0; i < numInputs; i++) { + values->data[i] = psVectorAlloc(numOrdinates, PS_TYPE_F32); + if (norm) { + pmReadout *ro = inputs->data[i]; // Readout of interest + float normValue; // Normalisation value + if (!ordinateLookup(&normValue, normConcept, false, NAN, NAN, ro)) { + psWarning("Unable to find value of %s for readout %d", normConcept, i); + roMask->data.U8[i] = 0xff; + norm->data.F32[i] = NAN; + numBadInputs++; + continue; + } + if (normValue == 0.0) { + psWarning("Normalisation value (%s) for readout %d is zero", normConcept, i); + roMask->data.U8[i] = 0xff; + norm->data.F32[i] = NAN; + numBadInputs++; + continue; + } + norm->data.F32[i] = 1.0 / normValue; + } + } + psVector *orders = psVectorAlloc(numOrdinates, PS_TYPE_U8); // Orders for each concept + for (int i = 0; i < numOrdinates; i++) { + pmDarkOrdinate *ord = ordinates->data[i]; // Ordinate information + if (ord->order <= 0) { + psError(PS_ERR_UNKNOWN, true, "Bad order for concept %s (%d) --- ignored", ord->name, ord->order); + psFree(values); + psFree(roMask); + psFree(orders); + return false; + } + orders->data.U8[i] = ord->order; + + // Mask the readout and move on + #define MASK_READOUT_VALUE { \ + roMask->data.U8[j] = 0xff; \ + val->data.F32[i] = NAN; \ + numBadInputs++; \ + continue; \ + } + + for (int j = 0; j < numInputs; j++) { + psVector *val = values->data[j]; // Value vector for readout + if (roMask->data.U8[j]) { + val->data.F32[i] = NAN; + continue; + } + + pmReadout *ro = inputs->data[j]; // Readout of interest + float value = NAN; // Value of ordinate + if (!ordinateLookup(&value, ord->name, ord->scale, ord->min, ord->max, ro)) { + roMask->data.U8[j] = 0xff; + val->data.F32[i] = NAN; + numBadInputs++; + continue; + } + val->data.F32[i] = value; + } + } + + if (psTraceGetLevel("psModules.detrend") > 9) { + for (int i = 0; i < numInputs; i++) { + psVector *val = values->data[i]; + (void) val; // avoid unused variable message when tracing is compiled out + for (int j = 0; j < numOrdinates; j++) { + psTrace("psModules.detrend", 9, "Image %d, ordinate %d: %f\n", i, j, val->data.F32[j]); + } + } + } + + psPolynomialMD *poly = psPolynomialMDAlloc(orders); // Polynomial for fitting + psFree(orders); + int numTerms = poly->coeff->n; // Number of terms in polynomial + if (numTerms > numInputs - numBadInputs) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Insufficient inputs (%d) to fit polynomial terms (%d).", + numInputs - numBadInputs, numTerms); + psFree(values); + psFree(roMask); + psFree(norm); + return false; + } + + // Set up output + int minInputCols, maxInputCols, minInputRows, maxInputRows; // Smallest and largest values to combine + int xSize, ySize; // Size of the output image + if (!pmReadoutStackValidate(&minInputCols, &maxInputCols, &minInputRows, &maxInputRows, &xSize, &ySize, + inputs)) { + psError(PS_ERR_UNKNOWN, false, "No valid input readouts."); + psFree(values); + psFree(roMask); + psFree(norm); + return false; + } + if (output->readouts->n != numTerms) { + output->readouts = psArrayRealloc(output->readouts, numTerms); + } + int outRow0 = 0, outCol0 = 0; // Output row0 and col0 + for (int i = 0; i < numTerms; i++) { + pmReadout *ro = output->readouts->data[i]; // Readout to update + if (!ro) { + ro = output->readouts->data[i] = pmReadoutAlloc(output); + } + + pmReadoutUpdateSize(ro, minInputCols, minInputRows, xSize, ySize, false, false, 0); + if (i == 0) { + outRow0 = ro->row0; + outCol0 = ro->col0; + } else { + assert(ro->row0 == outRow0); + assert(ro->col0 == outCol0); + } + } + + psImage *counts = pmReadoutAnalysisImage(output->readouts->data[0], PM_READOUT_STACK_ANALYSIS_COUNT, + xSize, ySize, PS_TYPE_U16, 0); + if (!counts) { + return false; + } + psImage *sigma = pmReadoutAnalysisImage(output->readouts->data[0], PM_READOUT_STACK_ANALYSIS_SIGMA, + xSize, ySize, PS_TYPE_F32, NAN); + if (!sigma) { + psFree(counts); + return false; + } + + // Iterate over pixels, fitting polynomial + psVector *pixels = psVectorAlloc(numInputs, PS_TYPE_F32); // Stack of pixels + psVector *mask = psVectorAlloc(numInputs, PS_TYPE_MASK); // Mask for stack + for (int i = minInputRows; i < maxInputRows; i++) { + int yOut = i - outRow0; // y position on output readout +#ifdef SHOW_BUSY + if (psTraceGetLevel("psModules.detrend") > 9) { + printf("Processing row %d\r", i); + fflush(stdout); + } +#endif + + for (int j = minInputCols; j < maxInputCols; j++) { + int xOut = j - outCol0; // x position on output readout + + psVectorInit(mask, 0); + for (int r = 0; r < inputs->n; r++) { + if (roMask->data.U8[r]) { + mask->data.PS_TYPE_MASK_DATA[r] = 0xff; + continue; + } + pmReadout *readout = inputs->data[r]; // Input readout + int yIn = i - readout->row0; // y position on input readout + int xIn = j - readout->col0; // x position on input readout + + pixels->data.F32[r] = readout->image->data.F32[yIn][xIn]; + if (norm) { + pixels->data.F32[r] *= norm->data.F32[r]; + } + if (readout->mask) { + mask->data.PS_TYPE_MASK_DATA[r] = readout->mask->data.PS_TYPE_MASK_DATA[yIn][xIn]; + } + } + + if (!psPolynomialMDClipFit(poly, pixels, NULL, mask, maskVal, values, iter, rej)) { + psErrorClear(); // Nothing we can do about it + psVectorInit(poly->coeff, NAN); + } + for (int k = 0; k < numTerms; k++) { + pmReadout *ro = output->readouts->data[k]; // Readout of interest + ro->image->data.F32[yOut][xOut] = poly->coeff->data.F64[k]; + } + counts->data.U16[yOut][xOut] = poly->numFit; + sigma->data.F32[yOut][xOut] = poly->stdevFit; + } + } + + psFree(norm); + psFree(roMask); + psFree(poly); + psFree(pixels); + psFree(mask); + psFree(values); + + psMetadataAddPtr(output->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_ORDINATES, + PS_DATA_ARRAY | PS_META_REPLACE, "Dark ordinates", ordinates); + psMetadataAddStr(output->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_NORM, + PS_META_REPLACE, "Dark normalisation", normConcept); + + for (int i = 0; i < numTerms; i++) { + pmReadout *ro = output->readouts->data[i]; // Readout to update + ro->data_exists = true; + } + output->data_exists = true; + output->parent->data_exists = true; + + return true; +} + + + +bool pmDarkApply(pmReadout *readout, const pmCell *dark, psMaskType bad) +{ + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_PTR_NON_NULL(dark, false); + PS_ASSERT_IMAGE_NON_NULL(readout->image, false); + PS_ASSERT_IMAGE_TYPE(readout->image, PS_TYPE_F32, false); + int numCols = readout->image->numCols, numRows = readout->image->numRows; // Size of image + if (readout->mask) { + PS_ASSERT_IMAGE_NON_NULL(readout->mask, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(readout->mask, readout->image, false); + PS_ASSERT_IMAGE_TYPE(readout->mask, PS_TYPE_MASK, false); + } + int numTerms = dark->readouts->n; // Number of polynomial terms + for (int i = 0; i < numTerms; i++) { + pmReadout *ro = dark->readouts->data[i]; // Dark readout + PS_ASSERT_PTR_NON_NULL(ro, false); + PS_ASSERT_IMAGE_NON_NULL(ro->image, false); + PS_ASSERT_IMAGE_SIZE(ro->image, numCols, numRows, false); + PS_ASSERT_IMAGE_TYPE(ro->image, PS_TYPE_F32, false); + } + psArray *ordinates = psMetadataLookupPtr(NULL, dark->analysis, PM_DARK_ANALYSIS_ORDINATES); // Ordinates + if (!ordinates) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find dark ordinates."); + return false; + } + bool mdok; // Status of MD lookup + psString normConcept = psMetadataLookupStr(&mdok, dark->analysis, PM_DARK_ANALYSIS_NORM); // Normalisation + + int numOrdinates = ordinates->n; // Number of ordinates + psVector *values = psVectorAlloc(numOrdinates, PS_TYPE_F32); // Values of ordinates + for (int i = 0; i < numOrdinates; i++) { + pmDarkOrdinate *ord = ordinates->data[i]; // Ordinate of interest + float value = NAN; // Value for ordinate + if (!ordinateLookup(&value, ord->name, ord->scale, ord->min, ord->max, readout)) { + psError(PS_ERR_UNKNOWN, true, "Unable to find value for %s", ord->name); + psFree(values); + return false; + } + values->data.F32[i] = value; + } + float norm = NAN; // Normalisation value + bool doNorm = false; // Do normalisation? + if (normConcept && strlen(normConcept) > 0) { + if (!ordinateLookup(&norm, normConcept, false, NAN, NAN, readout)) { + psError(PS_ERR_UNKNOWN, true, "Unable to find value for %s", normConcept); + psFree(values); + return false; + } + doNorm = true; + } + + psVector *orders = psVectorAlloc(numOrdinates, PS_TYPE_U8); // Order for each polynomial + for (int i = 0; i < numOrdinates; i++) { + pmDarkOrdinate *ord = ordinates->data[i]; // Ordinate information + if (ord->order <= 0) { + psError(PS_ERR_UNKNOWN, true, "Bad order for concept %s (%d) --- ignored", ord->name, ord->order); + psFree(values); + psFree(orders); + return false; + } + orders->data.U8[i] = ord->order; + } + + psPolynomialMD *poly = psPolynomialMDAlloc(orders); // Polynomial to apply + psFree(orders); + + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + for (int i = 0; i < numTerms; i++) { + pmReadout *ro = dark->readouts->data[i]; // Dark readout + poly->coeff->data.F64[i] = ro->image->data.F32[y][x]; + } + float value = psPolynomialMDEval(poly, values); // Value of dark current + if (doNorm) { + value *= norm; + } + readout->image->data.F32[y][x] -= value; + if (readout->mask && !isfinite(value)) { + readout->mask->data.PS_TYPE_MASK_DATA[y][x] = bad; + } + } + } + + psFree(poly); + psFree(values); + + return true; +} + +bool pmFPAWriteDark(pmFPA *fpa, psFits *fits, psDB *db, bool blank, bool recurse) +{ + PS_ASSERT_PTR_NON_NULL(fpa, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + if (!pmFPAWrite(fpa, fits, db, blank, recurse)) { + psError(PS_ERR_IO, false, "Unable to write FPA dark images"); + return false; + } + + pmHDU *hdu = fpa->hdu; + if (blank || !hdu) { + // No more to do + return true; + } + + psArray *ordinates = NULL; // Dark ordinates, to write + const char *normConcept = NULL; // Normalisation concept + psArray *chips = fpa->chips; // Component chips + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + psArray *cells = chip->cells; // Component cells + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; // Cell of interest + bool mdok; // Status of MD lookup + psArray *newOrd = psMetadataLookupPtr(&mdok, cell->analysis, PM_DARK_ANALYSIS_ORDINATES); // Ords + if (!mdok) { + continue; + } + psString newNorm = psMetadataLookupPtr(&mdok, cell->analysis, PM_DARK_ANALYSIS_NORM); // Norm + if (ordinates) { + if (newOrd != ordinates) { + psError(PS_ERR_UNKNOWN, true, "Dark ordinates differ across cells."); + return false; + } + if ((normConcept && (!newNorm || strcmp(normConcept, newNorm) != 0)) || + (!normConcept && newNorm)) { + psError(PS_ERR_UNKNOWN, true, "Dark normalisations differ across cells."); + return false; + } + } else { + ordinates = newOrd; + normConcept = newNorm; + } + } + } + + if (!ordinates) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find dark ordinates."); + return false; + } + + return pmDarkWrite(fits, hdu->header, ordinates, normConcept); +} + +bool pmChipWriteDark(pmChip *chip, psFits *fits, psDB *db, bool blank, bool recurse) +{ + PS_ASSERT_PTR_NON_NULL(chip, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + if (!pmChipWrite(chip, fits, db, blank, recurse)) { + psError(PS_ERR_IO, false, "Unable to write chip dark images"); + return false; + } + + pmHDU *hdu = pmHDUFromChip(chip); + if (blank || !hdu) { + // No more to do + return true; + } + + psArray *ordinates = NULL; // Dark ordinates, to write + const char *normConcept = NULL; // Normalisation concept + psArray *cells = chip->cells; // Component cells + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; // Cell of interest + bool mdok; // Status of MD lookup + psArray *newOrd = psMetadataLookupPtr(&mdok, cell->analysis, PM_DARK_ANALYSIS_ORDINATES); // Ordinates + if (!mdok) { + continue; + } + psString newNorm = psMetadataLookupPtr(&mdok, cell->analysis, PM_DARK_ANALYSIS_NORM); // Normalisation + if (ordinates) { + if (newOrd != ordinates) { + psError(PS_ERR_UNKNOWN, true, "Dark ordinates differ across cells."); + return false; + } + if ((normConcept && (!newNorm || strcmp(normConcept, newNorm) != 0)) || + (!normConcept && newNorm)) { + psError(PS_ERR_UNKNOWN, true, "Dark normalisations differ across cells."); + return false; + } + } else { + ordinates = newOrd; + normConcept = newNorm; + } + } + + if (!ordinates) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find dark ordinates."); + return false; + } + + return pmDarkWrite(fits, hdu->header, ordinates, normConcept); +} + +bool pmCellWriteDark(pmCell *cell, psFits *fits, psDB *db, bool blank) +{ + PS_ASSERT_PTR_NON_NULL(cell, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + // Allow the usual pmFPAWrite functions to handle the heavy lifting for the images + if (!pmCellWrite(cell, fits, db, blank)) { + psError(PS_ERR_UNKNOWN, false, "Unable to write dark cell."); + return false; + } + + pmHDU *hdu = pmHDUFromCell(cell); + if (blank || !hdu) { + // No more to do + return true; + } + + psArray *ordinates = psMetadataLookupPtr(NULL, cell->analysis, PM_DARK_ANALYSIS_ORDINATES); + if (!ordinates) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find dark ordinates."); + return false; + } + bool mdok; // Status of MD lookup + psString normConcept = psMetadataLookupPtr(&mdok, cell->analysis, PM_DARK_ANALYSIS_NORM); // Normalisation + + return pmDarkWrite(fits, hdu->header, ordinates, normConcept); +} + +bool pmDarkWrite(psFits *fits, psMetadata *header, const psArray *ordinates, const char *normConcept) +{ + PS_ASSERT_FITS_NON_NULL(fits, false); + PS_ASSERT_FITS_WRITABLE(fits, false); + PS_ASSERT_ARRAY_NON_NULL(ordinates, false); + + // Only write table once per FITS file + bool write = false; // Write table? + if (psFitsGetSize(fits) <= 1) { + write = true; + } else { + psMetadata *headers = psFitsReadHeaderSet(NULL, fits); // FITS headers + if (!psMetadataLookup(headers, PM_DARK_FITS_EXTNAME)) { + write = true; + } + psFree(headers); + } + if (!write) { + return true; + } + + if (!psMemIncrRefCounter((psMetadata*)header)) { + header = psMetadataAlloc(); + } + psMetadataAddStr(header, PS_LIST_TAIL, PM_DARK_HEADER_NORM, PS_META_REPLACE, + "Dark normalisation concept", normConcept); + + if (ordinates->n > 0) { + // Format ordinates into FITS table + int numOrdinates = ordinates->n;// Number of ordinates + psArray *table = psArrayAlloc(numOrdinates); // FITS table, constructed from ordinates + for (int i = 0; i < ordinates->n; i++) { + pmDarkOrdinate *ord = ordinates->data[i]; // Ordinate of interest + psMetadata *row = psMetadataAlloc(); // FITS table row + psMetadataAddStr(row, PS_LIST_TAIL, PM_DARK_FITS_NAME, 0, "Concept name", ord->name); + psMetadataAddS32(row, PS_LIST_TAIL, PM_DARK_FITS_ORDER, 0, "Polynomial order", ord->order); + psMetadataAddBool(row, PS_LIST_TAIL, PM_DARK_FITS_SCALE, 0, "Scale values?", ord->scale); + psMetadataAddF32(row, PS_LIST_TAIL, PM_DARK_FITS_MIN, 0, "Minimum value", ord->min); + psMetadataAddF32(row, PS_LIST_TAIL, PM_DARK_FITS_MAX, 0, "Maximum value", ord->max); + table->data[i] = row; + } + + if (!psFitsWriteTable(fits, header, table, PM_DARK_FITS_EXTNAME)) { + psError(PS_ERR_IO, false, "Unable to write dark ordinates."); + psFree(table); + psFree(header); + return false; + } + psFree(table); + } else { + // No ordinates to write to a table, so write to a blank header. + if (!psFitsWriteBlank(fits, header, PM_DARK_FITS_EXTNAME)) { + psError(PS_ERR_IO, false, "Unable to write dark header."); + psFree(header); + return false; + } + } + psFree(header); + + return true; +} + + +bool pmFPAReadDark(pmFPA *fpa, psFits *fits, psDB *db) +{ + PS_ASSERT_PTR_NON_NULL(fpa, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + if (!pmFPARead(fpa, fits, db)) { + psError(PS_ERR_UNKNOWN, false, "Unable to read dark FPA."); + return false; + } + + psString normConcept = NULL; // Normalisation concept + psArray *ordinates = pmDarkRead(&normConcept, fits); // Dark ordinates + if (!ordinates) { + psError(PS_ERR_IO, false, "Unable to read dark ordinates."); + return false; + } + + psArray *chips = fpa->chips; // Component chips + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + psArray *cells = chip->cells; // Component cells + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; // Cell of interest + psMetadataAddPtr(cell->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_ORDINATES, + PS_DATA_ARRAY | PS_META_REPLACE, "Dark ordinates", ordinates); + psMetadataAddStr(cell->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_NORM, PS_META_REPLACE, + "Dark normalisation", normConcept); + } + } + psFree(ordinates); + psFree(normConcept); + + return true; +} + +bool pmChipReadDark(pmChip *chip, psFits *fits, psDB *db) +{ + PS_ASSERT_PTR_NON_NULL(chip, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + if (!pmChipRead(chip, fits, db)) { + psError(PS_ERR_UNKNOWN, false, "Unable to read dark chip."); + return false; + } + + psString normConcept = NULL; // Normalisation concept + psArray *ordinates = pmDarkRead(&normConcept, fits); // Dark ordinates + if (!ordinates) { + psError(PS_ERR_IO, false, "Unable to read dark ordinates."); + return false; + } + + psArray *cells = chip->cells; // Component cells + for (int i = 0; i < cells->n; i++) { + pmCell *cell = cells->data[i]; // Cell of interest + psMetadataAddPtr(cell->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_ORDINATES, + PS_DATA_ARRAY | PS_META_REPLACE, "Dark ordinates", ordinates); + psMetadataAddStr(cell->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_NORM, PS_META_REPLACE, + "Dark normalisation", normConcept); + } + psFree(ordinates); + psFree(normConcept); + + return true; +} + + +bool pmCellReadDark(pmCell *cell, psFits *fits, psDB *db) +{ + PS_ASSERT_PTR_NON_NULL(cell, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + // Allow the usual pmFPARead functions to handle the heavy lifting for the images + if (!pmCellRead(cell, fits, db)) { + psError(PS_ERR_UNKNOWN, false, "Unable to read dark cell."); + return false; + } + + psString normConcept = NULL; // Normalisation concept + psArray *ordinates = pmDarkRead(&normConcept, fits); // Dark ordinates + if (!ordinates) { + psError(PS_ERR_IO, false, "Unable to read dark ordinates."); + return false; + } + psMetadataAddPtr(cell->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_ORDINATES, + PS_DATA_ARRAY | PS_META_REPLACE, "Dark ordinates", ordinates); + psMetadataAddStr(cell->analysis, PS_LIST_TAIL, PM_DARK_ANALYSIS_NORM, PS_META_REPLACE, + "Dark normalisation", normConcept); + psFree(ordinates); + psFree(normConcept); + + return true; +} + +psArray *pmDarkRead(psString *normConcept, psFits *fits) +{ + PS_ASSERT_PTR_NON_NULL(normConcept, NULL); + PS_ASSERT_PTR_NULL(*normConcept, NULL); + PS_ASSERT_FITS_NON_NULL(fits, NULL); + + if (!psFitsMoveExtName(fits, PM_DARK_FITS_EXTNAME)) { + psError(PS_ERR_IO, false, "Unable to find extension with dark ordinates table (%s).", + PM_DARK_FITS_EXTNAME); + return false; + } + + psMetadata *header = psFitsReadHeader(NULL, fits); // Header + bool mdok; // Status of MD lookup + *normConcept = psMemIncrRefCounter(psMetadataLookupStr(&mdok, header, PM_DARK_HEADER_NORM)); + + psArray *ordinates = NULL; // Dark ordinates to return + + psFitsType type = psFitsGetExtType(fits); // Type of FITS extension + switch (type) { + case PS_FITS_TYPE_IMAGE: { + // Check that it's of zero size; otherwise we might have some conflict + int numCols = psMetadataLookupS32(&mdok, header, "NAXIS1"); + int numRows = psMetadataLookupS32(&mdok, header, "NAXIS2"); + if (numCols != 0 || numRows != 0) { + psError(PS_ERR_UNKNOWN, true, "Extension %s is not a DARK table.", PM_DARK_FITS_EXTNAME); + psFree(header); + return NULL; + } + // No ordinates fit --- only a constant term + ordinates = psArrayAlloc(0); + break; + } + case PS_FITS_TYPE_BINARY_TABLE: + case PS_FITS_TYPE_ASCII_TABLE: { + psArray *table = psFitsReadTable(fits); // FITS Table with ordinates + int numOrdinates = table->n; // Number of ordinates + ordinates = psArrayAlloc(numOrdinates); + + for (int i = 0; i < numOrdinates; i++) { + psMetadata *row = table->data[i]; // Row of interest + const char *name = psMetadataLookupStr(NULL, row, PM_DARK_FITS_NAME); // Concept name + int order = psMetadataLookupS32(NULL, row, PM_DARK_FITS_ORDER); // Polynomial order + if (!name || order <= 0) { + psError(PS_ERR_UNKNOWN, false, "Bad value reading dark ordinates table."); + psFree(table); + psFree(ordinates); + return false; + } + pmDarkOrdinate *ord = pmDarkOrdinateAlloc(name, order); // Ordinate data + ord->scale = psMetadataLookupBool(NULL, row, PM_DARK_FITS_SCALE); + ord->min = psMetadataLookupF32(NULL, row, PM_DARK_FITS_MIN); + ord->max = psMetadataLookupF32(NULL, row, PM_DARK_FITS_MAX); + + ordinates->data[i] = ord; + } + psFree(table); + break; + } + default: + psError(PS_ERR_UNKNOWN, true, "Unrecognised FITS extension type."); + return NULL; + } + psFree(header); + + return ordinates; +} + Index: /tags/pap_merge_030828/psModules/src/detrend/pmDark.h =================================================================== --- /tags/pap_merge_030828/psModules/src/detrend/pmDark.h (revision 17232) +++ /tags/pap_merge_030828/psModules/src/detrend/pmDark.h (revision 17232) @@ -0,0 +1,98 @@ +#ifndef PM_DARK_H +#define PM_DARK_H + +#include +#include +#include + +#define PM_DARK_ANALYSIS_ORDINATES "DARK.ORDINATES" // Name for dark ordinates in the cell analysis metadata +#define PM_DARK_ANALYSIS_NORM "DARK.NORM" // Name for dark normalisation concept in cell analysis metadata +#define PM_DARK_HEADER_NORM "PSDRKNRM" // Header keyword for dark normalisation concept + +// An ordinate for fitting darks +typedef struct { + psString name; // Name of concept to fit + int order; // Polynomial order to fit + bool scale; // Rescale values? + float min, max; // Minimum and maximum values for rescaling +} pmDarkOrdinate; + +// Allocator +pmDarkOrdinate *pmDarkOrdinateAlloc(const char *name, // Name for ordinate + int order // Order for ordinate + ); + + +// Combine darks +bool pmDarkCombine(pmCell *output, // Output cell; readouts will be attached + const psArray *inputs, // Input readouts for combination + psArray *ordinates, // Ordinates for fitting + const char *normConcept, // Concept name to use to divide input pixel values + int iter, // Number of rejection iterations + float rej, // Rejection threshold (standard deviations) + psMaskType maskVal // Value to mask + ); + +// Apply dark +bool pmDarkApply(pmReadout *readout, // Readout to which to apply dark + const pmCell *dark, // Dark to apply + psMaskType bad // Mask value to give bad pixels + ); + +// I/O functions for darks + +// Write all darks within an FPA +bool pmFPAWriteDark(pmFPA *fpa, // FPA to write + psFits *fits, // FITS file to which to write + psDB *db, // Database, for concepts + bool blank, // Write a blank only? + bool recurse // Recurse to lower levels? + ); + +// Write all darks within a chip +bool pmChipWriteDark(pmChip *chip, // Chip to write + psFits *fits, // FITS file to which to write + psDB *db, // Database, for concepts + bool blank, // Write a blank only? + bool recurse // Recurse to lower levels? + ); + +// Write a dark to a FITS file +bool pmCellWriteDark(pmCell *cell, // Cell containing dark information + psFits *fits, // FITS file to which to write + psDB *db, // Database, for concepts + bool blank // Write a blank only? + ); + +// Read dark for all FPA from a FITS file +bool pmFPAReadDark(pmFPA *fpa, // FPA for which to read + psFits *fits, // FITS file to read + psDB *db // Database, for concepts + ); + +// Read dark for all chip from a FITS file +bool pmChipReadDark(pmChip *chip, // Chip for which to read + psFits *fits, // FITS file to read + psDB *db // Database, for concepts + ); + +// Read dark for a cell from a FITS file +bool pmCellReadDark(pmCell *cell, // Cell for which to read + psFits *fits, // FITS file to read + psDB *db // Database, for concepts + ); + +// Write dark table to FITS file +bool pmDarkWrite(psFits *fits, // FITS file to which to write + psMetadata *header, // Header to write + const psArray *ordinates, // Dark ordinates to write + const char *normConcept // Normalisation concept name + ); + +// Read dark table from FITS file +psArray *pmDarkRead(psString *normConcept, // Normalisation concept name + psFits *fits // FITS file to read + ); + + +#endif Index: /tags/pap_merge_030828/psModules/src/detrend/pmMaskBadPixels.c =================================================================== --- /tags/pap_merge_030828/psModules/src/detrend/pmMaskBadPixels.c (revision 17232) +++ /tags/pap_merge_030828/psModules/src/detrend/pmMaskBadPixels.c (revision 17232) @@ -0,0 +1,269 @@ +#if HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmHDUUtils.h" +#include "pmFPAMaskWeight.h" +#include "pmMaskBadPixels.h" + +bool pmMaskBadPixels(pmReadout *input, const pmReadout *mask, psMaskType maskVal) +{ + PS_ASSERT_PTR_NON_NULL(input, false); + PS_ASSERT_PTR_NON_NULL(input->mask, false); + PS_ASSERT_IMAGE_TYPE(input->mask, PS_TYPE_MASK, false); + + PS_ASSERT_PTR_NON_NULL(mask, false); + PS_ASSERT_PTR_NON_NULL(mask->mask, false); + PS_ASSERT_IMAGE_TYPE(mask->mask, PS_TYPE_MASK, false); + + psImage *inMask = input->mask; + psImage *exMask = mask->mask; + + // Add mask MD5 to header + pmHDU *hdu = pmHDUFromReadout(input); // HDU of interest + psVector *md5 = psImageMD5(mask->mask); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psStringPrepend(&md5string, "MASK image MD5: "); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + md5string, ""); + psFree(md5string); + + int rowMax = input->row0 + inMask->numRows; + int colMax = input->col0 + inMask->numCols; + + if (mask->row0 > input->row0 || mask->col0 > input->col0 || + mask->row0 + exMask->numRows < rowMax || mask->col0 + exMask->numCols < colMax) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Input image size exceeds that of mask image: (%d, %d) vs (%d, %d)", + inMask->numRows, inMask->numCols, exMask->numRows, exMask->numCols); + return false; + } + + // Determine total offset based on image offset with chip offset + // XXX if we choose to correct for the readout location, apply input->col0,row0 here + int offCol = input->col0 - mask->col0; + int offRow = input->row0 - mask->row0; + + // masks are both of type PS_TYPE_MASK + psMaskType **exVal = exMask->data.U8; + psMaskType **inVal = inMask->data.U8; + + // apply exMask values + if (maskVal) { + // set raised pixels in exMask which are selected by maskVal + for (int j = 0; j < inMask->numRows; j++) { + int xJ = j - offRow; + for (int i = 0; i < inMask->numCols; i++) { + int xI = i - offCol; + inVal[j][i] |= (maskVal & exVal[xJ][xI]); + } + } + } + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now, used for reporting + psString timeString = psTimeToISO(time); // String with time + psFree(time); + psStringPrepend(&timeString, "Static mask (selecting %x) applied at ", maskVal); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + timeString, ""); + psFree(timeString); + + return true; +} + + +bool pmMaskFlagSuspectPixels(pmReadout *output, const pmReadout *readout, float median, float stdev, + float rej, psMaskType maskVal) +{ + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_FLOAT_LARGER_THAN(rej, 0.0, false); + PS_ASSERT_IMAGE_NON_NULL(readout->image, false); + PS_ASSERT_IMAGE_NON_EMPTY(readout->image, false); + PS_ASSERT_IMAGE_TYPE(readout->image, PS_TYPE_F32, false); + if (readout->mask) { + PS_ASSERT_IMAGE_NON_EMPTY(readout->mask, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(readout->image, readout->mask, false); + PS_ASSERT_IMAGE_TYPE(readout->mask, PS_TYPE_MASK, false); + } + PS_ASSERT_PTR_NON_NULL(output, false); + + bool mdok; // Status of MD lookup + psImage *suspect = psMetadataLookupPtr(&mdok, output->analysis, PM_MASK_ANALYSIS_SUSPECT); // Suspect img + if (suspect) { + PS_ASSERT_IMAGE_NON_EMPTY(suspect, false); + PS_ASSERT_IMAGE_TYPE(suspect, PS_TYPE_S32, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(readout->image, suspect, false); + psMemIncrRefCounter(suspect); + } else { + suspect = psImageAlloc(readout->image->numCols, readout->image->numRows, PS_TYPE_S32); + psImageInit(suspect, 0); + psMetadataAddImage(output->analysis, PS_LIST_TAIL, PM_MASK_ANALYSIS_SUSPECT, PS_META_REPLACE, + "Suspect pixels", suspect); + psMetadataAddS32(output->analysis, PS_LIST_TAIL, PM_MASK_ANALYSIS_NUM, PS_META_REPLACE, + "Number of input images", 0); + } + + if (!isfinite(median) || !isfinite(stdev)) { + // If we get down here and the statistics are missing, then we should go and mask the entire image + psWarning("Missing statistics --- flagging entire image as suspect."); + return (psImage*)psBinaryOp(suspect, suspect, "+", psScalarAlloc(1.0, PS_TYPE_S32)); + } + + psImage *image = readout->image; // Image of interest + psImage *mask = readout->mask; // Corresponding mask + + psTrace ("psModules.detrend", 3, "suspect: %f +/- %f\n", median, stdev); + + for (int y = 0; y < image->numRows; y++) { + for (int x = 0; x < image->numCols; x++) { + if (fabs((image->data.F32[y][x] - median) / stdev) >= rej && + (!mask || !(mask->data.PS_TYPE_MASK_DATA[y][x] & maskVal))) { + suspect->data.S32[y][x]++; + } + } + } + psFree(suspect); // Drop reference + + psMetadataItem *numItem = psMetadataLookup(output->analysis, PM_MASK_ANALYSIS_NUM); // Item with number + assert(numItem); + numItem->data.S32++; + + return true; +} + +bool pmMaskIdentifyBadPixels(pmReadout *output, psMaskType maskVal, float thresh, pmMaskIdentifyMode mode) +{ + PS_ASSERT_PTR_NON_NULL(output, false); + psImage *suspects = psMetadataLookupPtr(NULL, output->analysis, PM_MASK_ANALYSIS_SUSPECT); // Suspect img + if (!suspects) { + psError(PS_ERR_UNKNOWN, false, "Unable to find image with suspected bad pixels."); + return false; + } + PS_ASSERT_IMAGE_NON_EMPTY(suspects, false); + PS_ASSERT_IMAGE_TYPE(suspects, PS_TYPE_S32, false); + if (output->mask) { + PS_ASSERT_IMAGE_NON_EMPTY(output->mask, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(output->mask, suspects, false); + PS_ASSERT_IMAGE_TYPE(output->mask, PS_TYPE_MASK, false); + } else { + output->mask = psImageAlloc(suspects->numCols, suspects->numRows, PS_TYPE_MASK); + } + int num = psMetadataLookupS32(NULL, output->analysis, PM_MASK_ANALYSIS_NUM); // Number of inputs + PS_ASSERT_INT_POSITIVE(num, false); + + float limit = NAN; // Limit for masking + switch (mode) { + case PM_MASK_ID_VALUE: + limit = thresh; + break; + + case PM_MASK_ID_FRACTION: + limit = thresh * num; + break; + + case PM_MASK_ID_SIGMA: { + psStats *stats = psStatsAlloc(PS_STAT_CLIPPED_STDEV); // Statistics + stats->clipSigma = 5.0; + stats->clipIter = 1; + if (!psImageStats(stats, suspects, NULL, 0) || !isfinite(stats->clippedStdev)) { + psError(PS_ERR_UNKNOWN, false, "Unable to perform statistics.\n"); + psFree(stats); + return NULL; + } + limit = thresh * stats->clippedStdev; + psTrace ("psModules.detrend", 3, "bad: %f -> %f\n", stats->clippedStdev, limit); + psFree(stats); + break; + } + + case PM_MASK_ID_POISSON: { + psStats *stats = psStatsAlloc(PS_STAT_MAX); // Statistics + if (!psImageStats(stats, suspects, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "Unable to perform statistics.\n"); + psFree(stats); + return NULL; + } + psHistogram *histo = psHistogramAlloc(-0.5, stats->max + 0.5, stats->max + 1); + psFree(stats); + if (!psImageHistogram(histo, suspects, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate histogram.\n"); + psFree(histo); + return NULL; + } + + // Find the mode. Since this is a Poisson distribution (more or less), this should also be the mean + // and variance. + int max = 0; // Index of the mode + for (int i = 0; i < histo->nums->n; i++) { + if (histo->nums->data.F32[i] > histo->nums->data.F32[max]) { + max = i; + } + } + + // Since the mode is most likely zero, we add one to get something realistic. Then "thresh" is + // negative, so we subtract instead of add. + limit = max + 1.0 - thresh * sqrtf((float)max + 1.0); + + psTrace ("psModules.detrend", 3, "bad: mode: %d, stdev: %f, limit: %f\n", + max, sqrtf((float)max + 1.0), limit); + break; + } + default: + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Invalid mask identify mode"); + return NULL; + } + + if (psTraceGetLevel("psModules.detrend") > 9) { + psStats *stats = psStatsAlloc(PS_STAT_MIN | PS_STAT_MAX); // Statistics + psImageStats(stats, suspects, NULL, 0); + psHistogram *histo = psHistogramAlloc(-0.5, stats->max + 0.5, stats->max + 1); + psImageHistogram(histo, suspects, NULL, 0); + for (int i = 0; i < histo->nums->n; i++) { + printf("%f --> %f : %f\n", histo->bounds->data.F32[i], histo->bounds->data.F32[i + 1], + histo->nums->data.F32[i]); + } + psFree(stats); + psFree(histo); + printf("Threshold: %f\n", limit); + } + + psTrace ("psModules.detrend", 3, "bad pixel threshold: %f", limit); + + psImage *badpix = output->mask; // Bad pixel mask + psImageInit(badpix, 0); + + for (int y = 0; y < suspects->numRows; y++) { + for (int x = 0; x < suspects->numCols; x++) { + if (suspects->data.S32[y][x] >= limit) { + badpix->data.PS_TYPE_MASK_DATA[y][x] = maskVal; + } + } + } + + return true; +} + +pmMaskIdentifyMode pmMaskIdentifyModeFromString (const char *string) { + + if (!strcasecmp(string, "VALUE")) { + return PM_MASK_ID_VALUE; + } + if (!strcasecmp(string, "FRACTION")) { + return PM_MASK_ID_FRACTION; + } + if (!strcasecmp(string, "SIGMA")) { + return PM_MASK_ID_SIGMA; + } + if (!strcasecmp(string, "POISSON")) { + return PM_MASK_ID_POISSON; + } + return PM_MASK_ID_NONE; +} Index: /tags/pap_merge_030828/psModules/src/detrend/pmMaskBadPixels.h =================================================================== --- /tags/pap_merge_030828/psModules/src/detrend/pmMaskBadPixels.h (revision 17232) +++ /tags/pap_merge_030828/psModules/src/detrend/pmMaskBadPixels.h (revision 17232) @@ -0,0 +1,71 @@ +/* @file pmMaskBadPixels.h + * @brief Mask bad pixels + * + * @author Ross Harman, MHPCC + * @author Eugene Magnier, IfA + * + * @version $Revision: 1.16 $ $Name: not supported by cvs2svn $ + * @date $Date: 2008-03-29 03:10:17 $ + * Copyright 2004 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_MASK_BAD_PIXELS_H +#define PM_MASK_BAD_PIXELS_H + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +#define PM_MASK_ANALYSIS_SUSPECT "MASK.SUSPECT" // Readout analysis metadata keyword for suspect image +#define PM_MASK_ANALYSIS_NUM "MASK.NUM" // Readout analysis metadata keyword for number of inputs + + +typedef enum { + PM_MASK_ID_NONE, + PM_MASK_ID_VALUE, + PM_MASK_ID_FRACTION, + PM_MASK_ID_SIGMA, + PM_MASK_ID_POISSON, +} pmMaskIdentifyMode; + +pmMaskIdentifyMode pmMaskIdentifyModeFromString (const char *string); + +/// Applies the bad pixel mask to the input +/// +/// Pixels marked as bad within the mask are marked as bad within the input image's mask. If maskVal is +/// non-zero, all pixels in the mask have any of the same bits sets as maskVal shall have the corresponding +/// bits raised. If maskVal is zero, any zero pixels in the mask are OR-ed with PM_MASK_BAD. Position +/// offsets (such as due to trimming) between the input and mask are applied so that the same pixels are +/// referred to. The science readout must already have a supplied mask element (use eg. pmReadoutSetMask). +/// The supplied mask image must be of MASK type +bool pmMaskBadPixels(pmReadout *input, ///< Input science image + const pmReadout *mask, ///< Mask image to apply + psMaskType maskVal ///< Mask value to apply + ); + +/// Find pixels outlying from the background, flagging suspect pixels +/// +/// Pixels more than "rej" standard deviations from the background level (in flat-fielded, +/// background-subtracted images) have the corresponding pixel in the "suspect pixels" image +/// incremented. After accumulating over a suitable sample of images, bad pixels should have a +/// high value in the suspect pixels image, allowing them to be identified. The suspect pixels +/// image is of type S32. The relevant median and standard deviation must be supplied in the +/// readout->analysis metadata as READOUT.MEDIAN, READOUT.STDEVe +bool pmMaskFlagSuspectPixels(pmReadout *output, ///< Output readout, optionally with suspect pixels image + const pmReadout *readout, ///< Readout to inspect + float median, ///< Image median + float stdev, ///< Image standard deviation + float rej, ///< Rejection threshold (standard deviations) + psMaskType maskVal ///< Mask value for statistics + ); + +/// Identify bad pixels from the suspect pixels image +/// +/// Bad pixels are identified from the suspect pixels image (accumulated over a large number of images), +/// according to the chosen mode. +bool pmMaskIdentifyBadPixels(pmReadout *output, ///< Output readout, with suspect pixels imageOut + psMaskType maskVal, ///< Value to set for bad pixels + float thresh, ///< Threshold for bad pixel + pmMaskIdentifyMode mode ///< Mode for identifying bad pixels + ); +/// @} +#endif Index: /tags/pap_merge_030828/psModules/src/detrend/pmShutterCorrection.c =================================================================== --- /tags/pap_merge_030828/psModules/src/detrend/pmShutterCorrection.c (revision 17232) +++ /tags/pap_merge_030828/psModules/src/detrend/pmShutterCorrection.c (revision 17232) @@ -0,0 +1,1023 @@ +#if HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmHDUUtils.h" +#include "psVectorBracket.h" +#include "pmConceptsAverage.h" +#include "pmReadoutStack.h" + +#include "pmShutterCorrection.h" + +/// Measure shutter correction: +/// +/// input : collection of shutter correction exposures (pre-processed) +/// output : a shutter correction image +/// +/// The measurement could be performed on any focal-plane unit at a time. for GPC, the obvious scale is to +/// measure the effect on the entire focal plane at once, with a single reference point in the field. this is +/// a little more complex than just measuring the effect for a single 2D image array. the reference point and +/// the detailed analysis points need to be defined for the entire hierarchy rather than just as coordinate +/// pairs or regions. a pmFPAview would be a natural element with which to define these points, but at the +/// moment, the pmFPAview structure defines a band in the CCD, not a coordinate. An option is to instead +/// specify the reference locations as a pmFPAview coupled with a psRegion, though we need to be careful not +/// to over-specify the pixels (ie, conflict between pmFPAview and psRegion). +/// +/// At each point in an image with exposure time T, we measure f(k;T) = F(k;T) / F(0;T) where k is the +/// coordinate of the point of interest, 0 is the reference coordinate, and F(k;T) is the measured number of +/// counts at the point of interest in this image. given a collection of f(k;T) values, we need to determine +/// the model f(k;T) = A(k) (T + dTk) / (T + dTo) where dTk is the shutter error at the given position, dTo is +/// the shutter error at the reference position, and A(k) is the scaling factor for the given position. +/// +/// The process for generating a shutter correction is as follows: +/// - for each image +/// -- measure the reference point counts +/// - for each analysis region: +/// -- measure shutter parameters (dTo, dTk, A): +/// --- for each image: +/// ---- measure counts at the region +/// ---- divide by the reference counts +/// --- linear extrapolation to find f(inf) = A(k) +/// --- linear extrapolation to find f(0) = A(k) dTk / dTo +/// --- linear interpolation to find coordinate where f(dTo) = A (1 + dTk/dTo) / 2 +/// --- non-linear fit of T, f(T) to f(k;T) = A(k) (T + dTk) / (T + dTo) +/// - use the collection of dTo values to choose a best value for dTo (median) +/// - for each image pixel +/// -- divide by the reference counts +/// -- generate the vectors T, f(T) +/// -- linear fit of T, f(T) to f(k;T) = A(k) (T + dTk) / (T + dTo) using dTo above +/// -- save dTk, A(k) in output image pixels +/// -- apply dTk, A(k) to measure residual images +/// -- generate residual FITS/JPEG images + + +#define MEASURE_SAMPLES 4 // Number of samples to make over the image. This should only be + // changed with great caution, since assumptions on its value are in + // the code (see pmShutterCorrectionDataAlloc). + + +static void pmShutterCorrectionFree(pmShutterCorrection *pars) +{ + // Nothing to free + return; +} + +pmShutterCorrection *pmShutterCorrectionAlloc() +{ + pmShutterCorrection *corr = (pmShutterCorrection*)psAlloc(sizeof(pmShutterCorrection)); + psMemSetDeallocator(corr, (psFreeFunc)pmShutterCorrectionFree); + + corr->scale = 0.0; + corr->offset = 0.0; + corr->offref = 0.0; + corr->num = 0; + corr->stdev = NAN; + + return corr; +} + +pmShutterCorrection *pmShutterCorrectionGuess(const psVector *exptime, const psVector *counts) +{ + // NOTE: vectors must be sorted on input. It is expensive to sort or check this here, but + // it is easy to arrange by sorting the images before generating these vectors. + + PS_ASSERT_VECTOR_NON_NULL(exptime, NULL); + PS_ASSERT_VECTOR_NON_NULL(counts, NULL); + PS_ASSERT_VECTOR_TYPE(exptime, PS_TYPE_F32, NULL); + PS_ASSERT_VECTOR_TYPE(counts, PS_TYPE_F32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(exptime, counts, NULL); + if (exptime->n <= 2) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Require more than 2 exposures to guess shutter correction.\n"); + return NULL; + } + + long N = exptime->n; // Number of exposures + + // use interpolation to guess shutter correction parameters given a set of exposures times and normalized + // counts (divided by the reference counts for each image) + + pmShutterCorrection *corr = pmShutterCorrectionAlloc(); // Shutter correction, to be returned + psPolynomial1D *line = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); // Straight line, for extrapolation + + // choose the highest exptime point as the guess for the scale: + // XXX we could examine the top 2 or 3 values and decide if we + // extended exptime enough or median clip. + corr->scale = counts->data.F32[N-1]; + + // fit a line to the lowest three points and extrapolate to 0.0 + psVector *tmpX = psVectorAlloc(2, PS_TYPE_F32); + psVector *tmpY = psVectorAlloc(2, PS_TYPE_F32); + + long index; + for (index = 0; !isfinite(exptime->data.F32[index]) && index < N - 1; index++) + ; // Iterate only + if (index == N - 1) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Not enough good values to guess shutter correction.\n"); + goto GUESS_ERROR; + } + tmpX->data.F32[0] = exptime->data.F32[index]; + tmpY->data.F32[0] = counts->data.F32[index]; + + for (index++; + (!isfinite(exptime->data.F32[index]) || exptime->data.F32[index] == exptime->data.F32[0]) && + index < N; index++) + ; // Iterate only + if (index == N) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Exposure times are all identical --- cannot guess shutter correction.\n"); + goto GUESS_ERROR; + } + tmpY->data.F32[1] = counts->data.F32[index]; + tmpX->data.F32[1] = exptime->data.F32[index]; + + // fit a line and extrapolate the fit to 0.0 + if (!psVectorFitPolynomial1D(line, NULL, 0, tmpY, NULL, tmpX)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to fit for the time offset.\n"); + goto GUESS_ERROR; + } + float ratio = psPolynomial1DEval(line, 0.0) / corr->scale; + + // XXX we need a sanity check: + // if the mean value of the three points is higher than corr->scale, + // then the slope should be negative. + // if the mean value of the three points is lower than corr->scale, + // then the slope should be positive. + + // find two points bracketing the value counts = A (1 + dTk/dTo) / 2 = corr->scale (1 + ratio) / 2 + float value = corr->scale * (1 + ratio) / 2.0; + + int Np; // Index of the value above (positive side) + if (ratio < 1.0) { + Np = psVectorBracket(counts, value, true); + } else { + Np = psVectorBracketDescend(counts, value, true); + } + int Nm = (Np == 0) ? 1 : Np - 1; // Index of the value below (negative side) + + tmpX->data.F32[0] = counts->data.F32[Nm]; + tmpX->data.F32[1] = counts->data.F32[Np]; + tmpY->data.F32[0] = exptime->data.F32[Nm]; + tmpY->data.F32[1] = exptime->data.F32[Np]; + + // fit a line and extrapolate the fit to counts = A (1 + dTk/dTo) : exptime = dTo + if (!psVectorFitPolynomial1D (line, NULL, 0, tmpY, NULL, tmpX)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to fit for the reference offset.\n"); + goto GUESS_ERROR; + } + corr->offref = psPolynomial1DEval(line, value); + corr->offset = ratio * corr->offref; + + psFree(line); + psFree(tmpX); + psFree(tmpY); + + return corr; + +GUESS_ERROR: + psFree(tmpX); + psFree(tmpY); + psFree(line); + psFree(corr); + return NULL; +} + +// linear fit to the counts and exptime, given a value for offref +pmShutterCorrection *pmShutterCorrectionLinFit(const psVector *exptime, const psVector *counts, + const psVector *cntError, const psVector *mask, float offref, + int nIter, float rej, psMaskType maskVal) +{ + PS_ASSERT_VECTOR_NON_NULL(exptime, NULL); + PS_ASSERT_VECTOR_NON_NULL(counts, NULL); + PS_ASSERT_VECTOR_TYPE(exptime, PS_TYPE_F32, NULL); + PS_ASSERT_VECTOR_TYPE(counts, PS_TYPE_F32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(exptime, counts, NULL); + if (exptime->n <= 2) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Require more than 2 exposures to guess shutter correction.\n"); + return NULL; + } + if (cntError) { + PS_ASSERT_VECTOR_TYPE(cntError, PS_TYPE_F32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(counts, cntError, NULL); + } + PS_ASSERT_FLOAT_LARGER_THAN(offref, 0.0, NULL); + + // this step is identical for all pixels: do it once and save? + psVector *x = psVectorAlloc(exptime->n, PS_TYPE_F32); + psVector *y = psVectorAlloc(exptime->n, PS_TYPE_F32); + + for (long i = 0; i < exptime->n; i++) { + // Should be safe (if expensive) to stick NaNs in --- the fitter deals with them + float value = 1.0 / (exptime->data.F32[i] + offref); + x->data.F32[i] = exptime->data.F32[i] * value; + y->data.F32[i] = value; + } + + psPolynomial2D *line = psPolynomial2DAlloc (PS_POLYNOMIAL_ORD, 1, 1); + + // mask out the terms we will not fit + line->coeffMask[0][0] = PS_POLY_MASK_SET; + line->coeffMask[1][1] = PS_POLY_MASK_SET; + line->coeff[0][0] = 0; + line->coeff[1][1] = 0; + + // the stats structure determines how the clipping statistic is measured + // too few points to use the robust analysis method + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV); + stats->clipSigma = rej; + stats->clipIter = nIter; + + if (!psVectorClipFitPolynomial2D(line, stats, mask, maskVal, counts, cntError, x, y)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to fit shutter correction.\n"); + psFree(stats); + psFree(x); + psFree(y); + psFree(line); + return NULL; + } + + pmShutterCorrection *corr = pmShutterCorrectionAlloc(); + corr->offref = offref; + corr->scale = line->coeff[1][0]; + corr->offset = line->coeff[0][1] / line->coeff[1][0]; + corr->num = stats->clippedNvalues; + corr->stdev = stats->clippedStdev; + + psFree(stats); + psFree(x); + psFree(y); + psFree(line); + + return corr; +} + +static psF32 pmShutterCorrectionModel(psVector *deriv, const psVector *params, const psVector *x) +{ + // This is in a tight loop, so we won't assert here. + + psF32 A = params->data.F32[0]; + psF32 p = x->data.F32[0] + params->data.F32[1]; + psF32 q = 1.0 / (x->data.F32[0] + params->data.F32[2]); + psF32 f = A * p * q; + + if (deriv) { + deriv->data.F32[0] = p * q; + deriv->data.F32[1] = A * q; + deriv->data.F32[2] = - f * q; + } + return f; +} + +// non-linear fit to the counts and exptime, given a guess for the three parameters +pmShutterCorrection *pmShutterCorrectionFullFit(const psVector *exptime, const psVector *counts, + const psVector *cntError, const pmShutterCorrection *guess) +{ + PS_ASSERT_VECTOR_NON_NULL(exptime, NULL); + PS_ASSERT_VECTOR_NON_NULL(counts, NULL); + PS_ASSERT_VECTOR_TYPE(exptime, PS_TYPE_F32, NULL); + PS_ASSERT_VECTOR_TYPE(counts, PS_TYPE_F32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(exptime, counts, NULL); + if (exptime->n <= 2) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Require more than 2 exposures to guess shutter correction.\n"); + return NULL; + } + if (cntError) { + PS_ASSERT_VECTOR_TYPE(cntError, PS_TYPE_F32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(counts, cntError, NULL); + } + PS_ASSERT_PTR_NON_NULL(guess, NULL); + + psMinimization *minInfo = psMinimizationAlloc(15, 0.1); // Minimization information + + psVector *params = psVectorAlloc (3, PS_TYPE_F32); // Fitting parameters + params->data.F32[0] = guess->scale; + params->data.F32[1] = guess->offset; + params->data.F32[2] = guess->offref; + + // XXX for the moment, don't set any constraints + // psMinConstraint *constraint = psMinConstraintAlloc(); + // constrain->checkLimits = pmShutterParamLimits; + // constrain->paramMask = NULL; + psMinConstraint *constraint = NULL; // Constraints on the minimization + + // XXX ignore covariance matrix for the moment + // psImage *covar = psImageAlloc (params->n, params->n, PS_TYPE_F64); + psImage *covar = NULL; // Covariance matrix + + // construct the coordinate and value entries (y is counts) + psArray *x = psArrayAlloc(exptime->n); // Coordinates + + for (long i = 0; i < exptime->n; i++) { + psVector *coord = psVectorAlloc(1, PS_TYPE_F32); + coord->data.F32[0] = exptime->data.F32[i]; + x->data[i] = coord; + } + + if (!psMinimizeLMChi2(minInfo, covar, params, constraint, x, counts, cntError, pmShutterCorrectionModel)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to fit for shutter correction.\n"); + psFree(x); + psFree(minInfo); + psFree(params); + return NULL; + } + + pmShutterCorrection *corr = pmShutterCorrectionAlloc(); // Shutter correction + corr->scale = params->data.F32[0]; + corr->offset = params->data.F32[1]; + corr->offref = params->data.F32[2]; + + psFree(minInfo); + psFree(params); + psFree(x); + + return corr; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +bool pmShutterCorrectionMeasure(pmReadout *output, const psArray *readouts, int size, psStatsOptions meanStat, + psStatsOptions stdevStat, int nIter, float rej, psMaskType maskVal) +{ + PS_ASSERT_ARRAY_NON_NULL(readouts, NULL); + PS_ASSERT_ARRAY_NON_EMPTY(readouts, NULL); + PS_ASSERT_INT_POSITIVE(size, NULL); + + long num = readouts->n; // Number of readouts + PS_ASSERT_INT_POSITIVE(nIter, NULL); + PS_ASSERT_FLOAT_LARGER_THAN(rej, 0.0, NULL); + + psArray *images = psArrayAlloc(num);// Array of images + psArray *masks = NULL; // Array of masks + psArray *weights = NULL; // Array of weights + psVector *exptimes = psVectorAlloc(num, PS_TYPE_F32); // Vector of exposure times + + { + pmReadout *readout = readouts->data[0]; // Representative readout + if (readout->mask) + { + masks = psArrayAlloc(num); + } + if (readout->weight) + { + weights = psArrayAlloc(num); + } + } + + // Check input sizes, generate first-pass statistics + psRegion refRegion; // Reference region + psVector *refs = psVectorAlloc(num, PS_TYPE_F32); // Reference measurements + psVectorInit(refs, 0); + psArray *regions = psArrayAlloc(MEASURE_SAMPLES); // Array of sample regions, made on each image + psImage *samplesMean = psImageAlloc(num, MEASURE_SAMPLES, PS_TYPE_F32); // Measurements for each file + psImage *samplesStdev = psImageAlloc(num, MEASURE_SAMPLES, PS_TYPE_F32); // Errors for each file + psStats *stats = psStatsAlloc(meanStat | stdevStat); + int numRows = 0, numCols = 0; // Size of images + for (long i = 0; i < images->n; i++) { + pmReadout *readout = readouts->data[i]; // Readout of interest + if (!readout) { + continue; + } + + bool mdok; // Status of MD lookup + float exptime = psMetadataLookupF32(&mdok, readout->parent->concepts, "CELL.EXPOSURE"); // Exp. time + if (!mdok || !isfinite(exptime)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure time for readout %ld is not set.\n", i); + goto MEASURE_ERROR; + } + exptimes->data.F32[i] = exptime; + + psImage *image = readout->image; // Image of interest + if (!image) { + continue; + } + images->data[i] = psMemIncrRefCounter(image); + if (image->type.type != PS_TYPE_F32) { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Bad type for image: %x\n", image->type.type); + goto MEASURE_ERROR; + } + if (numRows == 0 || numCols == 0) { + numRows = image->numRows; + numCols = image->numCols; + // Set up the sample regions + refRegion = psRegionForSquare(0.5 * numCols, 0.5 * numRows, size); + for (int j = 0; j < MEASURE_SAMPLES; j++) { + int x = (j % 2) ? size : image->numCols - size; + int y = (j > 1) ? size : image->numRows - size; + psRegion region = psRegionForSquare(x, y, size); + region = psRegionForImage(image, region); + regions->data[j] = psRegionAlloc(region.x0, region.x1, region.y0, region.y1); + } + } else if (numRows != image->numRows || numCols != image->numCols) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Image sizes don't match: %dx%d vs %dx%d\n", image->numCols, image->numRows, + numCols, numRows); + goto MEASURE_ERROR; + } + psImage *mask = readout->mask; // Mask of interest + if (mask) { + if (!masks) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Not all readouts have masks.\n"); + goto MEASURE_ERROR; + } + masks->data[i] = psMemIncrRefCounter(mask); + + if (mask->type.type != PS_TYPE_U8) { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Bad type for mask: %x\n", mask->type.type); + goto MEASURE_ERROR; + } + if (mask->numRows != numRows || mask->numCols != numCols) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Mask sizes don't match: %dx%d vs %dx%d\n", mask->numCols, mask->numRows, + numCols, numRows); + goto MEASURE_ERROR; + } + } else if (masks) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Not all readouts have masks.\n"); + goto MEASURE_ERROR; + } + + psImage *weight = readout->weight; // Weight map of interest + if (weight) { + if (!weights) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Not all readouts have weights.\n"); + goto MEASURE_ERROR; + } + weights->data[i] = psMemIncrRefCounter(weight); + + if (weight->type.type != PS_TYPE_F32) { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Bad type for weights: %x\n", weight->type.type); + goto MEASURE_ERROR; + } + if (weight->numRows != numRows || weight->numCols != numCols) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "Weight sizes don't match: %dx%d vs %dx%d\n", weight->numCols, weight->numRows, + numCols, numRows); + goto MEASURE_ERROR; + } + } else if (weights) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Not all readouts have weights.\n"); + goto MEASURE_ERROR; + } + + + // Measure statistics + if (!psImageStats(stats, image, mask, maskVal)) { + psWarning("Unable to measure reference statistics.\n"); + } + refs->data.F32[i] = psStatsGetValue(stats, meanStat); + psTrace("psModules.detrend", 3, "Reference value for image %ld = %f\n", i, refs->data.F32[i]); + if (refs->data.F32[i] <= 0.0) { + psError(PS_ERR_UNKNOWN, true, "Measured non-positive reference value.\n"); + goto MEASURE_ERROR; + } + refs->data.F32[i] = 1.0 / refs->data.F32[i]; + for (int j = 0; j < MEASURE_SAMPLES; j++) { + psRegion *region = regions->data[j]; // Region of interest + psImage *subImage = psImageSubset(image, *region); // Sub-image + psImage *subMask = NULL; + if (mask) { + subMask = psImageSubset(mask, *region); + } + if (!psImageStats(stats, subImage, subMask, maskVal)) { + psString regionString = psRegionToString(*region); + psWarning("Unable to measure sample statistics at %s in image %ld.\n", + regionString, i); + psFree(regionString); + } + psFree(subImage); + samplesMean->data.F32[j][i] = psStatsGetValue(stats, meanStat) * refs->data.F32[i]; + samplesStdev->data.F32[j][i] = psStatsGetValue(stats, stdevStat) * refs->data.F32[i]; + psTrace("psModules.detrend", 5, "Image %ld, sample %d: %f +/- %f\n", i, j, + samplesMean->data.F32[j][i], samplesStdev->data.F32[j][i]); + } + } + psFree(regions); + psFree(stats); + + float meanRef = 0.0; // Mean reference offset + int numGood = 0; // Number of good measurements + psVector *counts = psVectorAlloc(num, PS_TYPE_F32); // Mean for each image + psVector *errors = psVectorAlloc(num, PS_TYPE_F32); // Stdev for each image + for (int i = 0; i < MEASURE_SAMPLES; i++) { + counts = psImageRow(counts, samplesMean, i); + errors = psImageRow(errors, samplesStdev, i); + pmShutterCorrection *guess = pmShutterCorrectionGuess(exptimes, counts); // Guess at correction + pmShutterCorrection *corr = pmShutterCorrectionFullFit(exptimes, counts, errors, guess); // Correct'n + if (!corr) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to measure shutter reference correction.\n"); + psFree(guess); + psFree(counts); + psFree(errors); + goto MEASURE_ERROR; + } + psTrace("psModules.detrend", 5, "Sample reference value: %f\n", corr->offref); + if (isfinite(corr->offref)) { + meanRef += corr->offref; + numGood++; + } + psFree(corr); + psFree(guess); + } + psFree(samplesMean); + psFree(samplesStdev); + + if (numGood == 0) { + psError(PS_ERR_UNKNOWN, true, "Unable to measure mean reference offset.\n"); + psFree(counts); + psFree(errors); + goto MEASURE_ERROR; + } + meanRef /= (float)numGood; + psTrace("psModules.detrend", 3, "Mean reference value: %f\n", meanRef); + + // Check the weights + if (weights && nIter > 1) { + for (int i = 0; i < weights->n && nIter > 1; i++) { + psImage *weight = weights->data[i]; // Weight image + if (!weight) { + // We don't have weights, so no realistic errors: turn off iteration + if (nIter > 0) { + psWarning("Not all images have weights --- turning iteration off.\n"); + } + nIter = 1; + } + } + } + + psImage *shutter = psImageAlloc(numCols, numRows, PS_TYPE_F32); // Shutter correction image + psImage *pattern = psImageAlloc(numCols, numRows, PS_TYPE_F32); // Illumination pattern + psVector *mask = psVectorAlloc(num, PS_TYPE_U8); // Mask for each image + psVectorInit(mask, 0); + psTrace("psModules.detrend", 2, "Performing linear fit on individual pixels...\n"); + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + for (int i = 0; i < num; i++) { + psImage *image = images->data[i]; // Image of interest + counts->data.F32[i] = image->data.F32[y][x] * refs->data.F32[i]; + psImage *maskImage; // Mask image + if (masks && (maskImage = masks->data[i])) { + mask->data.U8[i] = maskImage->data.U8[y][x]; + } + psImage *weight; // Weight image + if (weights && (weight = weights->data[i])) { + errors->data.F32[i] = sqrtf(weight->data.F32[y][x]) * refs->data.F32[i]; + } else { + errors->data.F32[i] = sqrtf(image->data.F32[y][x]) * refs->data.F32[i]; + } + } + + pmShutterCorrection *corr = pmShutterCorrectionLinFit(exptimes, counts, errors, mask, meanRef, + nIter, rej, maskVal); + shutter->data.F32[y][x] = corr->offset; + pattern->data.F32[y][x] = corr->scale; + psFree(corr); + } + } + psFree(mask); + psFree(counts); + psFree(errors); + psFree(refs); + + if (psTraceGetLevel("psModules.detrend") > 5) { + psFits *fits = psFitsOpen("pattern.fits", "w"); + psFitsWriteImage(fits, NULL, pattern, 0, NULL); + psFitsClose(fits); + } + psFree(pattern); + + output->image = shutter; + + // Update the "concepts" + psList *inputCells = psListAlloc(NULL); // List of cells + for (long i = 0; i < readouts->n; i++) { + pmReadout *readout = readouts->data[i]; // Readout of interest + psListAdd(inputCells, PS_LIST_TAIL, readout->parent); + } + bool success = pmConceptsAverageCells(output->parent, inputCells, NULL, NULL, true); + psFree(inputCells); + + // Correct the exposure times --- they don't make sense any more. + psMetadataItem *item = psMetadataLookup(output->parent->concepts, "CELL.EXPOSURE"); + item->data.F32 = NAN; + item = psMetadataLookup(output->parent->concepts, "CELL.DARKTIME"); + item->data.F32 = NAN; + + return success; + + +MEASURE_ERROR: + // Clean up after error + psFree(exptimes); + psFree(images); + psFree(masks); + psFree(weights); + psFree(refs); + psFree(regions); + psFree(stats); + psFree(samplesMean); + psFree(samplesStdev); + return false; +} + + +bool pmShutterCorrectionApply(pmReadout *readout, const pmReadout *shutter, psMaskType blank) +{ + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_PTR_NON_NULL(shutter, false); + PS_ASSERT_IMAGE_NON_NULL(readout->image, false); + PS_ASSERT_IMAGE_NON_NULL(shutter->image, false); + PS_ASSERT_IMAGE_TYPE(readout->image, PS_TYPE_F32, false); + PS_ASSERT_IMAGE_TYPE(shutter->image, PS_TYPE_F32, false); + + psRegion region = psRegionSet(readout->col0, readout->col0 + readout->image->numCols, + readout->row0, readout->row0 + readout->image->numRows); // Detector region + + pmCell *cell = readout->parent; // Parent cell + if (!cell) { + psError(PS_ERR_BAD_PARAMETER_NULL, true, + "Parent cell is NULL --- unable to determine exposure time.\n"); + return false; + } + float exptime = psMetadataLookupF32(NULL, cell->concepts, "CELL.EXPOSURE"); // Exposure time + if (!isfinite(exptime)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Bad exposure time: %f.\n", exptime); + return false; + } + + pmHDU *hdu = pmHDUFromCell(cell);// HDU of interest + + psVector *md5 = psImageMD5(shutter->image); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psStringPrepend(&md5string, "Shutter image MD5: "); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, md5string, ""); + psFree(md5string); + + + psImage *shutterImage = psImageSubset(shutter->image, region); // Subimage with shutter + if (!shutterImage) { + psString regionString = psRegionToString(region); + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Size mismatch: %s vs %dx%d\n", + regionString, shutter->image->numCols, shutter->image->numRows); + psFree(regionString); + psFree(shutterImage); + return false; + } + psImage *image = readout->image; // Image to correct + psImage *mask = readout->mask; // Corresponding mask + + if (exptime <= 0.0) { + // In the extreme case that we have exptime <= 0.0, we correct the image to + // counts-per-second, rather than counts in the nominal exposure time + for (int y = 0; y < image->numRows; y++) { + for (int x = 0; x < image->numCols; x++) { + if (mask && !isfinite(shutterImage->data.F32[y][x])) { + mask->data.PS_TYPE_MASK_DATA[y][x] |= blank; + image->data.F32[y][x] = NAN; + continue; + } + image->data.F32[y][x] *= 1.0 / (exptime + shutterImage->data.F32[y][x]); + } + } + psMetadataAddF32 (cell->concepts, PS_LIST_TAIL, "CELL.EXPOSURE", PS_META_REPLACE, "exposure time re-normalized to 1.0", 1.0); // Exposure time + psString line = NULL; + psStringAppend (&line, "extreme exposure time %f, re-normalized to 1.0", exptime); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, line, ""); + psFree (line); + } else { + for (int y = 0; y < image->numRows; y++) { + for (int x = 0; x < image->numCols; x++) { + if (mask && !isfinite(shutterImage->data.F32[y][x])) { + mask->data.PS_TYPE_MASK_DATA[y][x] |= blank; + image->data.F32[y][x] = NAN; + continue; + } + image->data.F32[y][x] *= exptime / (exptime + shutterImage->data.F32[y][x]); + } + } + } + psFree(shutterImage); + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now, used for reporting + psString timeString = psTimeToISO(time); // String with time + psFree(time); + psStringPrepend(&timeString, "Shutter correction completed at "); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + timeString, ""); + psFree(timeString); + + + return true; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + +#define IMAGES_BUFFER 10 // Allocate space for this many images at a time + +static void shutterCorrectionDataFree(pmShutterCorrectionData *data) +{ + psFree(data->regions); + psFree(data->mean); + psFree(data->stdev); + + psFree(data->exptimes); + psFree(data->refs); + + return; +} + +pmShutterCorrectionData *pmShutterCorrectionDataAlloc(int numCols, int numRows, int size) +{ + pmShutterCorrectionData *data = psAlloc(sizeof(pmShutterCorrectionData)); + psMemSetDeallocator(data, (psFreeFunc)shutterCorrectionDataFree); + + data->num = 0; + data->numCols = 0; + data->numRows = 0; + + data->regions = psArrayAlloc(MEASURE_SAMPLES); + for (int j = 0; j < MEASURE_SAMPLES; j++) { + int x = (j % 2) ? size : numCols - size - 1; + int y = (j > 1) ? size : numRows - size - 1; + psRegion region = psRegionForSquare(x, y, size); + data->regions->data[j] = psRegionAlloc(region.x0, region.x1, region.y0, region.y1); + } + + data->mean = psArrayAlloc(MEASURE_SAMPLES); + data->stdev = psArrayAlloc(MEASURE_SAMPLES); + for (int i = 0; i < MEASURE_SAMPLES; i++) { + data->mean->data[i] = psVectorAllocEmpty(IMAGES_BUFFER, PS_TYPE_F32); + data->stdev->data[i] = psVectorAllocEmpty(IMAGES_BUFFER, PS_TYPE_F32); + } + + data->exptimes = psVectorAllocEmpty(IMAGES_BUFFER, PS_TYPE_F32); + data->refs = psVectorAllocEmpty(IMAGES_BUFFER, PS_TYPE_F32); + + return data; +} + +bool pmShutterCorrectionAddReadout(pmShutterCorrectionData *data, + const pmReadout *readout, ///< Readout to add + psStatsOptions meanStat, ///< Statistic to use for mean + psStatsOptions stdevStat, ///< Statistic to use for stdev + psMaskType maskVal, ///< Mask value + psRandom *rng ///< Random number generator + ) +{ + PS_ASSERT_PTR_NON_NULL(data, NULL); + PS_ASSERT_PTR_NON_NULL(readout, NULL); + PS_ASSERT_IMAGE_NON_NULL(readout->image, NULL); + PS_ASSERT_IMAGE_TYPE(readout->image, PS_TYPE_F32, NULL); + if (data->num == 0) { + data->numCols = readout->image->numCols; + data->numRows = readout->image->numRows; + } else { + PS_ASSERT_IMAGE_SIZE(readout->image, data->numCols, data->numRows, NULL); + } + if (readout->mask) { + PS_ASSERT_IMAGE_NON_NULL(readout->mask, NULL); + PS_ASSERT_IMAGE_TYPE(readout->mask, PS_TYPE_MASK, NULL); + PS_ASSERT_IMAGE_SIZE(readout->mask, data->numCols, data->numRows, NULL); + } + if (readout->weight) { + PS_ASSERT_IMAGE_NON_NULL(readout->weight, NULL); + PS_ASSERT_IMAGE_TYPE(readout->weight, PS_TYPE_F32, NULL); + PS_ASSERT_IMAGE_SIZE(readout->weight, data->numCols, data->numRows, NULL); + } + + // Add the exposure time + float exptime = psMetadataLookupF32(NULL, readout->parent->concepts, "CELL.EXPOSURE"); // Exp. time + if (!isfinite(exptime)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure time is not set."); + return false; + } + data->exptimes->data.F32[data->exptimes->n] = exptime; + data->exptimes = psVectorExtend(data->exptimes, IMAGES_BUFFER, 1); + + // Add the statistics + + // Add the reference value + psStats *stats = psStatsAlloc(meanStat | stdevStat); // Statistics to apply + if (!rng) { + rng = psRandomAlloc(PS_RANDOM_TAUS, 0); + } else { + psMemIncrRefCounter(rng); + } + if (!psImageBackground(stats, NULL, readout->image, readout->mask, maskVal, rng)) { + psError(PS_ERR_UNKNOWN, false, "Unable to measure reference statistics.\n"); + psFree(stats); + psFree(rng); + return false; + } + psFree(rng); + float refValue = psStatsGetValue(stats, meanStat); // Reference value + psTrace("psModules.detrend", 3, "Reference value for shutter image = %f\n", refValue); + if (refValue <= 0.0) { + psError(PS_ERR_UNKNOWN, true, "Measured non-positive reference value.\n"); + psFree(stats); + return false; + } + refValue = 1.0 / refValue; + data->refs->data.F32[data->refs->n] = refValue; + data->refs = psVectorExtend(data->refs, IMAGES_BUFFER, 1); + + // Add the region statistics + for (int j = 0; j < MEASURE_SAMPLES; j++) { + psRegion *region = data->regions->data[j]; // Region of interest + psRegion adjusted = *region; // Adjusted region, compensating for offsets + adjusted.x0 += readout->image->col0; + adjusted.x1 += readout->image->col0; + adjusted.y0 += readout->image->row0; + adjusted.y1 += readout->image->row0; + psImage *subImage = psImageSubset(readout->image, adjusted); // Sub-image + psImage *subMask = NULL; // Sub-image of mask + if (readout->mask) { + subMask = psImageSubset(readout->mask, adjusted); + } + if (!psImageStats(stats, subImage, subMask, maskVal)) { + psString regionString = psRegionToString(adjusted); + psWarning("Unable to measure sample statistics at %s in image.\n", + regionString); + psFree(regionString); + } + psFree(subImage); + psFree(subMask); + + psVector *mean = data->mean->data[j]; // Vector of means for this region + psVector *stdev = data->stdev->data[j]; // Vector of standard deviations for this region + + mean->data.F32[mean->n] = psStatsGetValue(stats, meanStat) * refValue; + stdev->data.F32[stdev->n] = psStatsGetValue(stats, stdevStat) * refValue; + data->mean->data[j] = psVectorExtend(mean, IMAGES_BUFFER, 1); + data->stdev->data[j] = psVectorExtend(stdev, IMAGES_BUFFER, 1); + } + + data->num++; + + return true; +} + +float pmShutterCorrectionReference(const pmShutterCorrectionData *data) +{ + PS_ASSERT_PTR_NON_NULL(data, NAN); + PS_ASSERT_INT_POSITIVE(data->num, NAN); + + float meanRef = 0.0; // Mean reference offset + int numGood = 0; // Number of good measurements + for (int i = 0; i < MEASURE_SAMPLES; i++) { + psVector *counts = data->mean->data[i]; // Mean for each image + psVector *errors = data->stdev->data[i]; // Stdev for each image + pmShutterCorrection *guess = pmShutterCorrectionGuess(data->exptimes, counts); // Guess at correction + pmShutterCorrection *corr = pmShutterCorrectionFullFit(data->exptimes, counts, + errors, guess); // The actual correction + psFree(guess); + if (corr && isfinite(corr->offref)) { + psTrace("psModules.detrend", 5, "Sample reference value: %f\n", corr->offref); + meanRef += corr->offref; + numGood++; + } + psFree(corr); + } + + if (numGood == 0) { + psError(PS_ERR_UNKNOWN, true, "Unable to measure mean reference offset.\n"); + return false; + } + meanRef /= (float)numGood; + psTrace("psModules.detrend", 3, "Mean reference value: %f\n", meanRef); + return meanRef; +} + +bool pmShutterCorrectionGenerate(pmReadout *shutter, pmReadout *pattern, const psArray *inputs, + float reference, const pmShutterCorrectionData *data, + int nIter, float rej, psMaskType maskVal) +{ + PS_ASSERT_PTR_NON_NULL(shutter, false); + PS_ASSERT_ARRAY_NON_NULL(inputs, false); + PS_ASSERT_INT_EQUAL(data->num, inputs->n, false); + PS_ASSERT_INT_NONNEGATIVE(nIter, false); + PS_ASSERT_FLOAT_LARGER_THAN(rej, 0.0, false); + + int minInputCols, maxInputCols, minInputRows, maxInputRows; // Smallest and largest values to combine + int xSize, ySize; // Size of the output image + if (!pmReadoutStackValidate(&minInputCols, &maxInputCols, &minInputRows, &maxInputRows, &xSize, &ySize, + inputs)) { + psError(PS_ERR_UNKNOWN, false, "No valid input readouts."); + return false; + } + + pmReadoutUpdateSize(shutter, minInputCols, minInputRows, xSize, ySize, false, false, maskVal); + if (pattern) { + pmReadoutUpdateSize(pattern, minInputCols, minInputRows, xSize, ySize, false, false, maskVal); + } + + psImage *nums = pmReadoutAnalysisImage(shutter, PM_READOUT_STACK_ANALYSIS_COUNT, xSize, ySize, + PS_TYPE_U16, 0); // Image with number fitted per pixel + if (!nums) { + return false; + } + psImage *sigma = pmReadoutAnalysisImage(shutter, PM_READOUT_STACK_ANALYSIS_SIGMA, xSize, ySize, + PS_TYPE_F32, NAN); // Image with stdev per pixel + if (!sigma) { + psFree(nums); + return false; + } + + psImage *shutterImage = shutter->image; // Shutter correction image + psImage *patternImage; // Illumination pattern + if (pattern) { + patternImage = psMemIncrRefCounter(pattern->image); + } else { + patternImage = psImageAlloc(xSize, ySize, PS_TYPE_F32); + } + + int num = data->num; // Number of images + psVector *counts = psVectorAlloc(num, PS_TYPE_F32); // Counts in each image + psVector *errors = psVectorAlloc(num, PS_TYPE_F32); // Counts in each image + psVector *mask = psVectorAlloc(num, PS_TYPE_MASK); // Mask for each image + psTrace("psModules.detrend", 2, "Performing linear fit on individual pixels...\n"); + for (int i = minInputRows; i < maxInputRows; i++) { + int yOut = i - shutter->row0; // y position on output readout + for (int j = minInputCols; j < maxInputCols; j++) { + int xOut = j - shutter->col0; // x position on output readout + + psVectorInit(mask, 0); + for (int r = 0; r < num; r++) { + pmReadout *readout = inputs->data[r]; // Readout of interest + int yIn = i - readout->row0; // y position on input readout + int xIn = j - readout->col0; // x position on input readout + psImage *image = readout->image; // Image of interest + float ref = data->refs->data.F32[r]; // (Inverse) reference value + counts->data.F32[r] = image->data.F32[yIn][xIn] * ref; + if (readout->mask) { + mask->data.PS_TYPE_MASK_DATA[r] = readout->mask->data.PS_TYPE_MASK_DATA[yIn][xIn]; + } + if (readout->weight) { + errors->data.F32[r] = sqrtf(readout->weight->data.F32[yIn][xIn]) * ref; + } else { + errors->data.F32[r] = sqrtf(image->data.F32[yIn][xIn]) * ref; + } + } + + pmShutterCorrection *corr = pmShutterCorrectionLinFit(data->exptimes, counts, errors, mask, + reference, nIter, rej, maskVal); + if (!corr) { + // Nothing we can do about it + psErrorClear(); + shutterImage->data.F32[yOut][xOut] = NAN; + patternImage->data.F32[yOut][xOut] = NAN; + nums->data.U16[yOut][xOut] = 0; + sigma->data.F32[yOut][xOut] = NAN; + continue; + } + shutterImage->data.F32[yOut][xOut] = corr->offset; + patternImage->data.F32[yOut][xOut] = corr->scale; + nums->data.U16[yOut][xOut] = corr->num; + sigma->data.F32[yOut][xOut] = corr->stdev; + psFree(corr); + } + } + psFree(mask); + psFree(errors); + psFree(counts); + psFree(nums); + psFree(sigma); + + // Update the "concepts" + psList *inputCells = psListAlloc(NULL); // List of cells + for (long i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + psListAdd(inputCells, PS_LIST_TAIL, readout->parent); + } + bool success = pmConceptsAverageCells(shutter->parent, inputCells, NULL, NULL, true); + psFree(inputCells); + + // Correct the exposure times --- they don't make sense any more. + psMetadataItem *item = psMetadataLookup(shutter->parent->concepts, "CELL.EXPOSURE"); + item->data.F32 = NAN; + item = psMetadataLookup(shutter->parent->concepts, "CELL.DARKTIME"); + item->data.F32 = NAN; + + shutter->data_exists = true; + shutter->parent->data_exists = true; + shutter->parent->parent->data_exists = true; + + if (pattern) { + pattern->data_exists = true; + pattern->parent->data_exists = true; + pattern->parent->parent->data_exists = true; + } + + return success; +} Index: /tags/pap_merge_030828/psModules/src/detrend/pmShutterCorrection.h =================================================================== --- /tags/pap_merge_030828/psModules/src/detrend/pmShutterCorrection.h (revision 17232) +++ /tags/pap_merge_030828/psModules/src/detrend/pmShutterCorrection.h (revision 17232) @@ -0,0 +1,195 @@ +/* @file pmShutterCorrection.h + * @brief Functions to build and apply a shutter exposure-time correction. + * + * @author Eugene Magnier, IfA + * @author Paul Price, IfA + * + * @version $Revision: 1.15 $ $Name: not supported by cvs2svn $ + * @date $Date: 2008-03-29 03:10:17 $ + * Copyright 2006 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_SHUTTER_CORRECTION_H +#define PM_SHUTTER_CORRECTION_H + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +/* A mechanical shutter may not yield uniform exposure times as a function of + * position on the detector. The typical error consists of a constant + * exposure-time offset relative to the requested value, ie exposure time is + * T_o + dT(x,y). The exposure error, dT, may be measured with the following + * scheme. Obtain a set of exposures with different exposures times taken of + * the same flat-field source; the source must be spatially stable between the + * exposures, but need not have a stable amplitude. For an illuminating flux + * of intensity F(x,y) = F_o f(x,y), the signal recorded by any pixel in the + * detector is given by: S(t,x,y) = F_o(t) f(x,y) (T_o + dT(x,y)) where F_o is + * the F_o(t) is the (variable) overall intensity of the illuminating source + * and f(x,y) is the spatial illumination patter times the flat-field response. + * Choose a reference location in the image (eg, the detector center) and + * divide by the value of that region (ie, mean or median): + * + * s(t,x,y) = S(t,x,y) / S(t,0,0) + * s(t,x,y) = F_o(t) f(x,y) (T_o + dT(x,y)) / F_o(t) f(0,0) (T_o + dT(0,0)) + * s(t,x,y) = f(x,y) (T_o + dT(x,y)) / f(0,0) (T_o + dT(0,0)) + * + * we can absorb the term f(0,0) into f(x,y) as we have no motivation for the + * scale of f(x,y). For any single pixel, over the set of exposures, we thus + * need to solve for dT(x,y), dT(0,0), and f'(x,y) in the equation: + * s(t,x,y) = f'(x,y) (T_o + dT(x,y)) / (T_o + dT(0,0)) + * + * we avoid directly fitting these values as the process would be a non-linear + * least-squares problem for every pixel in the image, and thus very time + * consuming. There are linear options which may be used instead. + * First, as T_o goes to a large value, s() approaches the value of f'(x,y). + * Next, as T_o goes to a very small value, s() approaches the value of + * f'(x,y)*dT(x,y)/dT(0,0). Finally, when s() has the value of + * f'(x,y)*(1 + dT(x,y)/dT(0,0))/2, T_o has the value of dT(0,0). with data + * points covering a reasonable dynamic range, we can solve for these three + * values by interpolation and/or extrapolation. + * + * To take the strategy one step further, we could use the above recipe to + * obtain a guess for the three parameters and then apply non-linear fitting to + * solve more accurately for the parameters. If we limit this operation to a + * handful of positions in the image (user defined, but the obvious choice would + * be positions near the center, edges, and corners), then we may determine a + * good value for dT(0,0). Since there is only one dT(0,0) for the image, we + * can apply the resulting measurement to the rest of the pixels in the image. + * If dT(0,0) is not a free parameter, then the fitting process is linear in + * terms of dT(x,y) and f'(x,y) + */ + +/// Shutter correction parameters, applicable for a single pixel +typedef struct { + double scale; ///< The normalisation for an exposure, A(k) + double offset; ///< The time offset, dTk + double offref; ///< The reference time offset, dTo + int num; ///< Number of points used + float stdev; ///< Standard deviation +} pmShutterCorrection; + +/// Allocator for shutter correction parameters +pmShutterCorrection *pmShutterCorrectionAlloc(); + +/// Guess a shutter correction, based on plot of counts vs exposure time +/// +/// This function is used before doing the full non-linear fit, to get parameters close to the true. Assumes +/// exptime vector is sorted (ascending order; longest is last) prior to input. +pmShutterCorrection *pmShutterCorrectionGuess( + const psVector *exptime, ///< Exposure times for each exposure + const psVector *counts ///< Counts for each exposure + ); + +/// Generate shutter correction based on a linear fit +/// +/// Performs a linear fit to counts as a function of exposure time, with the reference time offset fixed (so +/// that the system is linear). Performs iterative clipping, if nIter > 1. +pmShutterCorrection *pmShutterCorrectionLinFit( + const psVector *exptime, ///< Exposure times for each exposure + const psVector *counts, ///< Counts for each exposure + const psVector *cntError, ///< Error in the counts + const psVector *mask, ///< Mask for each exposure + float offref, ///< Reference time offset + int nIter, ///< Number of iterations + float rej, ///< Rejection threshold (sigma) + psMaskType maskVal ///< Mask value + ); + +/// Generate shutter correction based on a full non-linear fit +/// +/// Performs a full non-linear fit to counts as a function of exposure time. The main purpose is to solve for +/// the reference time offset, so that future fits may be performed using linear fitting with the reference +/// time offset fixed. +pmShutterCorrection *pmShutterCorrectionFullFit( + const psVector *exptime, ///< Exposure times for each exposure + const psVector *counts, ///< Counts for each exposure + const psVector *cntError, ///< Error in the counts + const pmShutterCorrection *guess ///< Initial guess + ); + +/// Measure a shutter correction image from an array of images +/// +/// Given an array of readouts (with known exposure times from the cell concepts), this function measures the +/// shutter correction (our principal concern is for the time offset, rather than the normalisation) by +/// measuring the reference time offset using the full non-linear fit for a small number of representative +/// regions (middle and corners), and then using that to perform a linear fit to each pixel. +bool pmShutterCorrectionMeasure( + pmReadout *output, ///< Output readout + const psArray *readouts, ///< Array of readouts + int size, ///< Size of samples for statistics for non-linear fit + psStatsOptions meanStat, ///< Statistic to use for mean + psStatsOptions stdevStat, ///< Statistic to use for stdev + int nIter, ///< Number of iterations + float rej, ///< Rejection threshold (sigma) + psMaskType maskVal ///< Mask value + ); + +/// Apply a shutter correction +/// +/// Given a shutter correction (with dT for each pixel), applies this correction to an input image. +bool pmShutterCorrectionApply( + pmReadout *readout, ///< Readout to which to apply shutter correction + const pmReadout *shutter, ///< Shutter correction readout, with dT for each pixel + psMaskType blank ///< Value to give blank pixels + ); + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Functions for doing the shutter correction piece-meal (don't have to read entire image stack into memory at +// once). A single read run through the stack is required, calling pmShutterCorrectionAddReadout on each. +// Then pmShutterCorrectionReference provides the required reference shutter time, so that +// pmShutterCorrectionGenerate can generate a shutter correction piece by piece as overlapping pixels from +// each input are read in. + + +/// Data for measuring the shutter correction +typedef struct { + int num; ///< Number of images + int numCols, numRows; ///< Size of images + psArray *regions; ///< Regions at which to measure statistics + psArray *mean; ///< Vector of means at each region + psArray *stdev; ///< Vector of standard deviations at each region + psVector *exptimes; ///< Exposure times for each image + psVector *refs; ///< Reference fluxes +} pmShutterCorrectionData; + +/// Allocator for pmShutterCorrectionData +pmShutterCorrectionData *pmShutterCorrectionDataAlloc(int numCols, int numRows, ///< Size of images + int size ///< Size of regions + ); + +/// Add a readout to the correction data +/// +/// Performs statistics on the readout, recording the data +bool pmShutterCorrectionAddReadout( + pmShutterCorrectionData *data, ///< Correction data + const pmReadout *readout, ///< Readout to add + psStatsOptions meanStat, ///< Statistic to use for mean + psStatsOptions stdevStat, ///< Statistic to use for stdev + psMaskType maskVal, ///< Mask value + psRandom *rng ///< Random number generator + ); + +/// Calculate the reference shutter time from the correction data +float pmShutterCorrectionReference( + const pmShutterCorrectionData *data ///< Correction data + ); + +/// Generate a shutter correction +/// +/// Performs the linear fit to each pixel in the stack. +bool pmShutterCorrectionGenerate( + pmReadout *shutter, ///< Shutter correction + pmReadout *pattern, ///< Background pattern (or NULL) + const psArray *inputs, ///< Stack of input pmReadouts + float reference, ///< Reference shutter time (from pmShutterCorrectionRef) + const pmShutterCorrectionData *data, ///< Correction data + int nIter, ///< Number of iterations + float rej, ///< Rejection threshold (sigma) + psMaskType maskVal ///< Mask value + ); + + + +/// @} +#endif Index: /tags/pap_merge_030828/psModules/src/imcombine/pmReadoutCombine.c =================================================================== --- /tags/pap_merge_030828/psModules/src/imcombine/pmReadoutCombine.c (revision 17232) +++ /tags/pap_merge_030828/psModules/src/imcombine/pmReadoutCombine.c (revision 17232) @@ -0,0 +1,375 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmHDUUtils.h" +#include "pmFPAMaskWeight.h" +#include "pmConceptsAverage.h" +#include "pmReadoutStack.h" + +#include "pmReadoutCombine.h" + +//#define SHOW_BUSY 1 // Show that the function is busy + + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Public functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Allocator for pmCombineParams +pmCombineParams *pmCombineParamsAlloc(psStatsOptions combine) +{ + pmCombineParams *params = psAlloc(sizeof(pmCombineParams)); + + params->combine = combine; + params->maskVal = 0; + params->blank = 0; + params->nKeep = 0; + params->fracHigh = 0.0; + params->fracHigh = 0.0; + params->iter = 1; + params->rej = INFINITY; + params->weights = false; + + return params; +} + + +// XXX: Maybe add support for S16 and S32 types. Currently, only F32 supported. +bool pmReadoutCombine(pmReadout *output, const psArray *inputs, const psVector *zero, const psVector *scale, + const pmCombineParams *params) +{ + // Check inputs + PS_ASSERT_PTR_NON_NULL(output, false); + PS_ASSERT_ARRAY_NON_NULL(inputs, false); + PS_ASSERT_PTR_NON_NULL(params, false); + if (zero) { + PS_ASSERT_VECTOR_TYPE(zero, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_SIZE(zero, inputs->n, false); + } + if (scale) { + PS_ASSERT_VECTOR_TYPE(scale, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_SIZE(scale, inputs->n, false); + } + PS_ASSERT_FLOAT_WITHIN_RANGE(params->fracLow, 0.0, 1.0, false); + PS_ASSERT_FLOAT_WITHIN_RANGE(params->fracHigh, 0.0, 1.0, false); + if (params->combine != PS_STAT_SAMPLE_MEAN && params->combine != PS_STAT_SAMPLE_MEDIAN && + params->combine != PS_STAT_ROBUST_MEDIAN && params->combine != PS_STAT_FITTED_MEAN && + params->combine != PS_STAT_CLIPPED_MEAN) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Combination method is not SAMPLE_MEAN, SAMPLE_MEDIAN, " + "ROBUST_MEDIAN, FITTED_MEAN or CLIPPED_MEAN.\n"); + return false; + } + for (int i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + if (params->weights && !readout->weight) { + psError(PS_ERR_UNEXPECTED_NULL, true, + "Rejection based on weights requested, but no weights supplied for image %d.\n", i); + return false; + } + } + + bool first = !output->image; // First pass through? + + pmHDU *hdu = pmHDUFromReadout(output); // Output HDU + if (!hdu) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find HDU for readout.\n"); + return false; + } + + if (first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Combining using statistic: %x", params->combine); + if (!hdu->header) { + hdu->header = psMetadataAlloc(); + } + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + psStatsOptions combineStdev = 0; // Statistics option for weights + switch (params->combine) { + case PS_STAT_SAMPLE_MEAN: + case PS_STAT_SAMPLE_MEDIAN: + combineStdev = PS_STAT_SAMPLE_STDEV; + break; + case PS_STAT_ROBUST_MEDIAN: + combineStdev = PS_STAT_ROBUST_STDEV; + break; + case PS_STAT_FITTED_MEAN: + combineStdev = PS_STAT_FITTED_STDEV; + break; + case PS_STAT_CLIPPED_MEAN: + combineStdev = PS_STAT_CLIPPED_STDEV; + break; + default: + psAbort("Should never get here --- checked params->combine before.\n"); + } + + psStats *stats = psStatsAlloc(params->combine | combineStdev); // The statistics to use in the combination + if (params->combine == PS_STAT_CLIPPED_MEAN) { + stats->clipSigma = params->rej; + stats->clipIter = params->iter; + + if (first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Combination clipping: %d iterations, rejection at %f sigma", + params->iter, params->rej); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + } + if (params->weights && first) { + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + "Using input weights to combine images", ""); + } + + int minInputCols, maxInputCols, minInputRows, maxInputRows; // Smallest and largest values to combine + int xSize, ySize; // Size of the output image + if (!pmReadoutStackValidate(&minInputCols, &maxInputCols, &minInputRows, &maxInputRows, &xSize, &ySize, + inputs)) { + psError(PS_ERR_UNKNOWN, false, "No valid input readouts."); + return false; + } + + pmReadoutUpdateSize(output, minInputCols, minInputRows, xSize, ySize, true, params->weights, + params->blank); + psTrace("psModules.imcombine", 7, "Output minimum: %d,%d\n", output->col0, output->row0); + + psImage *counts = pmReadoutAnalysisImage(output, PM_READOUT_STACK_ANALYSIS_COUNT, xSize, ySize, + PS_TYPE_U16, 0); + if (!counts) { + return false; + } + psImage *sigma = pmReadoutAnalysisImage(output, PM_READOUT_STACK_ANALYSIS_SIGMA, xSize, ySize, + PS_TYPE_F32, NAN); + if (!sigma) { + psFree(counts); + return false; + } + + stats->options |= combineStdev; + + // We loop through each pixel in the output image. We loop through each input readout. We determine if + // that output pixel is contained in the image from that readout. If so, we save it in psVector pixels. + // If not, we set a mask for that element in pixels. Then, we mask off pixels not between fracLow and + // fracHigh. Then we call the vector stats routine on those pixels/mask. Then we set the output pixel + // value to the result of the stats call. + + psVector *pixels = psVectorAlloc(inputs->n, PS_TYPE_F32); // Stack of pixels + psF32 *pixelsData = pixels->data.F32; // Dereference pixels + + psVector *mask = psVectorAlloc(inputs->n, PS_TYPE_U8); // Mask for stack + psU8 *maskData = mask->data.U8; // Dereference mask + + psVector *weights = NULL; // Stack of weights + psVector *errors = NULL; // Stack of errors (sqrt of variance/weights), for psVectorStats + psF32 *weightsData = NULL; // Dereference weights + if (params->weights) { + weights = psVectorAlloc(inputs->n, PS_TYPE_F32); // Stack of weights + weightsData = weights->data.F32; + } + psVector *index = NULL; // The indices to sort the pixels + + float keepFrac = 1.0 - params->fracLow - params->fracHigh; // Fraction of pixels to keep + if (keepFrac != 1.0 && first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Min/max rejection: %f high, %f low, keep %d", + params->fracHigh, params->fracLow, params->nKeep); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + psMaskType maskVal = params->maskVal; // The mask value + if (maskVal && first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Mask for combination: %x", maskVal); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + #ifndef PS_NO_TRACE + psTrace("psModules.imcombine", 3, "Iterating output: %d --> %d, %d --> %d\n", + minInputCols - output->col0, maxInputCols - output->col0, + minInputRows - output->row0, maxInputRows - output->row0); + if (psTraceGetLevel("psModules.imcombine") >= 3) { + for (int r = 0; r < inputs->n; r++) { + pmReadout *readout = inputs->data[r]; // Input readout + psTrace("psModules.imcombine", 3, "Iterating input %d: %d --> %d, %d --> %d\n", r, + minInputCols - readout->col0, maxInputCols - readout->col0, + minInputRows - readout->row0, maxInputRows - readout->row0); + } + } + #endif + + // Dereference output products + psF32 **outputImage = output->image->data.F32; // Output image + psU8 **outputMask = output->mask->data.U8; // Output mask + psF32 **outputWeight = NULL; // Output weight map + if (output->weight) { + outputWeight = output->weight->data.F32; + } + + psVector *invScale = NULL; // Inverse scale; pre-calculated for efficiency + if (scale) { + invScale = (psVector*)psBinaryOp(NULL, psScalarAlloc(1.0, PS_TYPE_F32), "/", (const psPtr)scale); + } + + for (int i = minInputRows; i < maxInputRows; i++) { + int yOut = i - output->row0; // y position on output readout + #ifdef SHOW_BUSY + + if (psTraceGetLevel("psModules.imcombine") > 9) { + printf("Processing row %d\r", i); + fflush(stdout); + } + #endif + for (int j = minInputCols; j < maxInputCols; j++) { + int xOut = j - output->col0; // x position on output readout + + int numValid = 0; // Number of valid pixels in the stack + memset(maskData, 0, mask->n * sizeof(psU8)); // Reset the mask + for (int r = 0; r < inputs->n; r++) { + pmReadout *readout = inputs->data[r]; // Input readout + int yIn = i - readout->row0; // y position on input readout + int xIn = j - readout->col0; // x position on input readout + psImage *image = readout->image; // The readout image + + #if 0 // This should have been taken care of already: + // Check bounds + if (xIn < 0 || xIn >= image->numCols || yIn < 0 || yIn >= image->numRows) { + continue; + } + #endif + + pixelsData[r] = image->data.F32[yIn][xIn]; + if (!isfinite(pixelsData[r])) { + maskData[r] = 1; + continue; + } + + // Check mask + psImage *roMask = readout->mask; // The mask image + if (roMask && roMask->data.U8[yIn][xIn] & maskVal) { + maskData[r] = 1; + continue; + } + + if (params->weights) { + weightsData[r] = readout->weight->data.F32[yIn][xIn]; + } + + if (zero) { + pixelsData[r] -= zero->data.F32[r]; + } + if (scale) { + pixelsData[r] *= invScale->data.F32[r]; + if (params->weights) { + weightsData[r] *= invScale->data.F32[r] * invScale->data.F32[r]; + } + } + + numValid++; + } + + if (numValid == 0) { + outputMask[yOut][xOut] = params->blank; + outputImage[yOut][xOut] = NAN; + counts->data.U16[yOut][xOut] = 0; + sigma->data.F32[yOut][xOut] = NAN; + continue; + } + + // Apply fracLow,fracHigh if there are enough pixels + if (numValid * keepFrac >= params->nKeep && keepFrac != 1.0) { + index = psVectorSortIndex(index, pixels); + int numLow = numValid * params->fracLow; // Number of low pixels to clip + int numHigh = numValid * params->fracHigh; // Number of high pixels to clip + // Low pixels + psS32 *indexData = index->data.S32; // Dereference index + for (int k = 0, numMasked = 0; numMasked < numLow && k < index->n; k++) { + // Don't count the ones that are already masked + if (!maskData[indexData[k]]) { + maskData[indexData[k]] = 1; + numMasked++; + numValid--; + } + } + // High pixels + for (int k = pixels->n - 1, numMasked = 0; numMasked < numHigh && k >= 0; k--) { + // Don't count the ones that are already masked + if (!maskData[indexData[k]]) { + maskData[indexData[k]] = 1; + numMasked++; + numValid--; + } + } + } + counts->data.U16[yOut][xOut] = numValid; + + // XXXXX this step probably is very expensive : convert errors to variance everywhere? + if (params->weights) { + errors = (psVector*)psUnaryOp(errors, weights, "sqrt"); + } + + // Combination + if (!psVectorStats(stats, pixels, errors, mask, 1)) { + // Can't do much about it, but it's not worth worrying about + psErrorClear(); + outputImage[yOut][xOut] = NAN; + outputMask[yOut][xOut] = params->blank; + if (params->weights) { + outputWeight[yOut][xOut] = NAN; + } + sigma->data.F32[yOut][xOut] = NAN; + } else { + outputImage[yOut][xOut] = psStatsGetValue(stats, params->combine); + outputMask[yOut][xOut] = isfinite(outputImage[yOut][xOut]) ? 0 : params->blank; + if (params->weights) { + float stdev = psStatsGetValue(stats, combineStdev); + outputWeight[yOut][xOut] = PS_SQR(stdev); // Variance + // XXXX this is not the correct formal error. + // also, the weighted mean is not obviously the correct thing here + } + sigma->data.F32[yOut][xOut] = psStatsGetValue(stats, combineStdev); + } + } + } + #ifdef SHOW_BUSY + if (psTraceGetLevel("psModules.imcombine") > 9) { + printf("\n"); + } + #endif + psFree(index); + psFree(pixels); + psFree(mask); + psFree(weights); + psFree(errors); + psFree(stats); + psFree(invScale); + psFree(counts); + psFree(sigma); + + // Update the "concepts" + psList *inputCells = psListAlloc(NULL); // List of cells + for (long i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + psListAdd(inputCells, PS_LIST_TAIL, readout->parent); + } + bool success = pmConceptsAverageCells(output->parent, inputCells, NULL, NULL, true); + psFree(inputCells); + + output->data_exists = true; + output->parent->data_exists = true; + output->parent->parent->data_exists = true; + + return success; +} + Index: /tags/pap_merge_030828/psModules/src/imcombine/pmReadoutCombine.h =================================================================== --- /tags/pap_merge_030828/psModules/src/imcombine/pmReadoutCombine.h (revision 17232) +++ /tags/pap_merge_030828/psModules/src/imcombine/pmReadoutCombine.h (revision 17232) @@ -0,0 +1,47 @@ +/* @file pmReadoutCombine.h + * @brief Combine multiple readouts + * + * @author George Gusciora, MHPCC + * @author Paul Price, IfA + * + * @version $Revision: 1.13 $ $Name: not supported by cvs2svn $ + * @date $Date: 2008-03-29 03:10:17 $ + * Copyright 2004-2006 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_READOUT_COMBINE_H +#define PM_READOUT_COMBINE_H + +/// @addtogroup imcombine Image Combinations +/// @{ + +/// Combination parameters for pmReadoutCombine. +/// +/// These values define how the combination is performed, and should not vary by detector, so that it can be +/// re-used for multiple combinations. +typedef struct { + psStatsOptions combine; ///< Statistic to use when performing the combination + psMaskType maskVal; ///< Mask value + psMaskType blank; ///< Mask value to give blank (i.e., no data) pixels + int nKeep; ///< Mimimum number of pixels to keep + float fracHigh; ///< Fraction of high pixels to immediately throw + float fracLow; ///< Fraction of low pixels to immediately throw + int iter; ///< Number of iterations for clipping (for CLIPPED_MEAN only) + float rej; ///< Rejection threshould for clipping (for CLIPPED_MEAN only) + bool weights; ///< Use the supplied weights (instead of calculated stdev)? +} pmCombineParams; + +// Allocator for pmCombineParams +pmCombineParams *pmCombineParamsAlloc(psStatsOptions statsOptions ///< Statistic to use for combination + ); + +/// Combine multiple readouts, applying zero and scale, with optional minmax clipping +bool pmReadoutCombine(pmReadout *output,///< Output readout; altered and returned + const psArray *inputs, ///< Array of input readouts (F32 image and weight, U8 mask) + const psVector *zero, ///< Zero corrections to subtract from input, or NULL + const psVector *scale, ///< Scale corrections to divide into input, or NULL + const pmCombineParams *params ///< Combination parameters + ); + +/// @} +#endif