Index: trunk/ippScripts/scripts/detrend_reject.pl
===================================================================
--- trunk/ippScripts/scripts/detrend_reject.pl	(revision 8194)
+++ trunk/ippScripts/scripts/detrend_reject.pl	(revision 8194)
@@ -0,0 +1,141 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use IPC::Cmd qw( can_run run );
+use PS::IPP::Metadata::Config;
+use Data::Dumper;
+
+use constant RECIPE => "PPSTATS_REJECT"; # Recipe to use on residuals
+use constant CLIP => 3;		# Rejection limit for sigma clipping
+
+# look for commands we need
+my $missing_tools;
+my $ppStats = can_run('ppStats')
+    or (warn "can't find ppStats" and $missing_tools = 1);
+die "Can't find required tools.\n" if $missing_tools;
+
+my @filenames = @ARGV;		# Input files
+my %stats;			# Statistics for each file
+
+my $mdcParser = PS::IPP::Metadata::Config->new;	# Parser for metadata config files
+
+foreach my $fileName (@filenames) {
+    my $command = "$ppStats $fileName -robust-median -robust-stdev"; # Command to run
+    print "Executing: $command\n";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => 0);
+    die "Unable to perform ppStats on $fileName: $error_code\n" if not $success;
+    my $parsed = $mdcParser->parse(join "", @$stdout_buf); # Parsed metadata
+    $stats{$fileName} = assembleStats($parsed);
+}
+
+my $rejectList = rejectFiles(\%stats); # List of files to reject
+
+foreach my $fileName (keys %$rejectList) {
+    print "$fileName\n";
+}
+
+### Pau.
+
+
+# Assemble the statistics into a Perl-ish data structure: a hash of hashes of hashes of hashes.
+sub assembleStats
+{
+    my $parsed = shift;		# Parsed metadata config file containing the stats
+
+    my %fpaStats;		# Container for statistics for fpa
+    foreach my $fpaItem (@$parsed) {
+	if ($fpaItem->{class} eq "metadata") {
+	    my $chipName = $fpaItem->{name}; # Name of chip
+	    my $chipData = $fpaItem->{value};
+	    my %chipStats;	# Container for statistics for chip
+	    foreach my $chipItem (@$chipData) {
+		if ($chipItem->{class} eq "metadata") {
+		    my $cellName = $chipItem->{name}; # Name of cell
+		    my $cellData = $chipItem->{value};
+		    my %cellStats; # Container for statistics for cell
+		    foreach my $cellItem (@$cellData) {
+			if ($cellItem->{name} =~ /^(SAMPLE|ROBUST|FITTED|CLIPPED)/) {
+			    # It's a statistic of some sort
+			    if ($cellItem->{name} =~ /STDEV$/) {
+				$cellStats{'stdev'} = $cellItem->{value};
+			    } else {
+				$cellStats{'value'} = $cellItem->{value};
+			    }
+			}
+		    }
+		    $chipStats{$cellName} = \%cellStats;
+		}
+	    }
+	    $fpaStats{$chipName} = \%chipStats;
+	}
+    }
+    return \%fpaStats;
+}
+
+# Reject files on the basis of the statistics
+sub rejectFiles
+{
+    my $stats = shift;		# The statistics structure
+
+    my %reject;			# List of files to reject
+
+    # Measure the mean
+    my $sum = 0;		# Sum of the values
+    my $num = 0;		# Number of values
+    foreach my $file (keys %$stats) {
+	my $fileData = $stats->{$file}; # Data for the file
+	foreach my $chipName (keys %$fileData) {
+	    my $chipData = $fileData->{$chipName}; # Data for the chip
+	    foreach my $cellName (keys %$chipData) {
+		my $cellData = $chipData->{$cellName}; # Data for the cell
+		if (defined $cellData->{'value'}) {
+		    $sum += $cellData->{'value'};
+		    $num++;
+		}
+	    }
+	}
+    }
+    die "Unable to measure mean\n" if $num == 0;
+    my $mean = $sum / $num;	# Mean of the values
+
+    # Measure the standard deviation
+    $sum = 0;
+    $num = 0;
+    foreach my $fileName (keys %$stats) {
+	my $fileData = $stats->{$fileName}; # Data for the file
+	foreach my $chipName (keys %$fileData) {
+	    my $chipData = $fileData->{$chipName}; # Data for the chip
+	    foreach my $cellName (keys %$chipData) {
+		my $cellData = $chipData->{$cellName}; # Data for the cell
+		if (defined $cellData->{'value'}) {
+		    $sum += ($cellData->{'value'} - $mean)**2;
+		    $num++;
+		}
+	    }
+	}
+    }
+    die "Unable to measure stadard deviation\n" if $num == 1;
+    my $stdev = sqrt($sum/($num-1)); # Standard deviation of the values
+
+    # Identify values exceeding the tolerance
+  FILE:    foreach my $fileName (keys %$stats) {
+      my $fileData = $stats->{$fileName}; # Data for the file
+      foreach my $chipName (keys %$fileData) {
+	  my $chipData = $fileData->{$chipName}; # Data for the chip
+	  foreach my $cellName (keys %$chipData) {
+	      my $cellData = $chipData->{$cellName}; # Data for the cell
+	      if (defined $cellData->{'value'}) {
+		  my $residual = abs($cellData->{'value'} - $mean) / $stdev;
+		  if ($residual > CLIP) {
+		      $reject{$fileName} = 1;
+		      next FILE;
+		  }
+	      }
+	  }
+      }
+  }
+    return \%reject;
+}