Index: /trunk/ippScripts/scripts/detrend_reject_imfile.pl
===================================================================
--- /trunk/ippScripts/scripts/detrend_reject_imfile.pl	(revision 8611)
+++ /trunk/ippScripts/scripts/detrend_reject_imfile.pl	(revision 8612)
@@ -7,4 +7,5 @@
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
+use Statistics::Descriptive;
 use Data::Dumper;
 
@@ -21,6 +22,6 @@
 # This measures how close it is to what's expected
 use constant REJECT_INDIVIDUAL_MEAN => {
-    'bias' => 0.01,		# Should be fairly flat; some CRs
-    'dark' => 0.05,		# Lots of CRs
+    'bias' => 5,		# Should be fairly flat; some CRs
+    'dark' => 10,		# Lots of CRs
     'flat' => undef		# Can't define expected value (depends on exposure level)
     };
@@ -29,7 +30,7 @@
 # This measures how much variation there is in each component
 use constant REJECT_INDIVIDUAL_STDEV => {
-    'bias' => 0.1,		# Should be fairly flat; some CRs
-    'dark' => 0.5,		# Lots of CRs
-    'flat' => 100		# Stars and galaxies
+    'bias' => 10,		# Should be fairly flat; some CRs
+    'dark' => 100,		# Lots of CRs
+    'flat' => 1000		# Stars and galaxies
     };
 
@@ -37,6 +38,6 @@
 # This measures how close it is to what's expected
 use constant REJECT_SAMPLE_MEAN => {
-    'bias' => 0.01,		# Should be little variation between chips
-    'dark' => 0.05,		# Could be some glow on some chips
+    'bias' => 10,		# Should be little variation between chips
+    'dark' => 100,		# Could be some glow on some chips
     'flat' => undef		# Can't define expected value (depends on exposure level)
     };
@@ -45,7 +46,7 @@
 # This measures how much variation there is across the components
 use constant REJECT_SAMPLE_STDEV => {
-    'bias' => 0.1,		# Should be little variation between chips
-    'dark' => 0.5,		# Could be some glow on some chips
-    'flat' => 10		# Could be features on some chips, but all should be about the same
+    'bias' => 10,		# Should be little variation between chips
+    'dark' => 100,		# Could be some glow on some chips
+    'flat' => 500		# Could be features on some chips, but all should be about the same
     };
 
@@ -71,5 +72,5 @@
 my $files;			# Array of component files
 {
-    my $command = "$dettool -residimfile -det_id $detId -iter $iter -exp_id $expId"; # Command to run
+    my $command = "$dettool -residimfile -det_id $detId -iteration $iter -exp_id $expId"; # Command to run
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 	run(command => $command, verbose => 1);
@@ -79,13 +80,20 @@
 }
 
-# Generate the file list
-my $outputName = $detId . '_' . $iter; # Root output name
-my $statName = $outputName . '.stats';
-my $listName = $outputName . '_jpeg.list'; # Name for the input file list
-open my $listFile, '>' . $listName;
+# Generate the file list, and get the statistics
+my $outputName = $detType . '_' . $detId . '_' . $iter; # Root output name
+my $list1Name = $outputName . '_jpeg1.list'; # Name for the input file list for binning 1
+my $list2Name = $outputName . '_jpeg2.list'; # Name for the input file list for binning 2
+my @means;			# Array of means
+my @stdevs;			# Array of stdevs
+open my $list1File, '>' . $list1Name;
+open my $list2File, '>' . $list2Name;
 foreach my $file (@$files) {
-    print $listFile $file->{uri} . "\n";
+    print $list1File $file->{b1_uri} . "\n";
+    print $list2File $file->{b2_uri} . "\n";
+    push @means, $file->{bg};
+    push @stdevs, $file->{bg_stdev};
 }
-close $listFile;
+close $list1File;
+close $list2File;
 
 # Output products --- need to synch with the camera configuration!
@@ -93,24 +101,18 @@
 my $jpeg2Name = $outputName . ".b2.jpeg"; # Binned JPEG #2
 
-# Make the jpeg
+# Make the jpeg for binning 1
 {
-    my $command = "$ppImage -list $listName $outputName -stat $statName" .
-	"-recipe PPIMAGE " .  RECIPE; # Command to run
+    my $command = "$ppImage -list $list1Name $outputName -recipe PPIMAGE " . RECIPE; # Command to run
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 	run(command => $command, verbose => 1);
-    die "Unable to find expected output file: $jpeg1Name\n" if not -f $jpeg1Name;
-    die "Unable to find expected output file: $jpeg2Name\n" if not -f $jpeg2Name;
-    die "Unable to find expected output file: $statName\n" if not -f $statName;
+#    die "Unable to find expected output file: $jpeg1Name\n" if not -f $jpeg1Name;
 }
 
-# Parse the statistics
-my $stats;			# Statistics from ppImage
+# Make the jpeg for binning 2
 {
-    open my $statsFile, $statName or die "Can't open statistics file $statName: $!\n";
-    my @contents = <$statsFile>; # Contents of file
-    close $statsFile;
-    my $metadata = $mdcParser->parse(join "", @contents); # Metadata from the stats
-    $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
-    $stats->parse($metadata) or die "Unable to find all values in statistics output.\n";
+    my $command = "$ppImage -list $list2Name $outputName -recipe PPIMAGE " . RECIPE; # Command to run
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => 1);
+#    die "Unable to find expected output file: $jpeg2Name\n" if not -f $jpeg2Name;
 }
 
@@ -128,6 +130,4 @@
 
 # Reject based on the individual stats
-my @means = $stats->bg_data();	# Array of means
-my @stdevs = $stats->bg_stdev_data(); # Array of stdevs
 my $reject = 0;			# Rejection flag
 die "Number of means and number of stdevs differ!\n" if scalar @means != scalar @stdevs;
@@ -137,7 +137,15 @@
     my $stdev = $stdevs[$i];	# Stdev for this component
 
-    if ((defined REJECT_INDIVIDUAL_MEAN->{$detType} and $mean / $stdev > REJECT_INDIVIDUAL_MEAN->{$detType})
-	or
-	(defined REJECT_INDIVIDUAL_STDEV->{$detType} and $stdev > REJECT_INDIVIDUAL_STDEV->{$detType})) {
+    if (defined REJECT_INDIVIDUAL_MEAN->{$detType}
+	and $stdev > 0 and
+	$mean / $stdev > REJECT_INDIVIDUAL_MEAN->{$detType}) {
+	print "Rejecting exposure based on bad individual mean for component $i: " .
+	    ($mean / $stdev) . " vs " . REJECT_INDIVIDUAL_MEAN->{$detType} . "\n";
+	$reject = 1;
+	last;
+    } elsif (defined REJECT_INDIVIDUAL_STDEV->{$detType} and
+	     $stdev > REJECT_INDIVIDUAL_STDEV->{$detType}) {
+	print "Rejecting exposure based on bad individual stdev for component $i: " .
+	    $stdev . " vs " . REJECT_INDIVIDUAL_STDEV->{$detType} . "\n";
 	$reject = 1;
 	last;
@@ -146,18 +154,30 @@
 
 # Reject based on the sample stats
-my $mean = $stats->bg_mean;	# Mean of the sample of means
-my $stdev = $stats->bg_stdev; # Stdev of the sample of means
-my $meanStdev = $stats->bg_mean_stdev; # Mean of the sample of stdevs
-
-if ((defined REJECT_SAMPLE_MEAN->{$detType} and $mean / $stdev > REJECT_SAMPLE_MEAN->{$detType})
-    or
-    (defined REJECT_SAMPLE_STDEV->{$detType} and $meanStdev > REJECT_SAMPLE_STDEV->{$detType})) {
+my $meanStats = Statistics::Descriptive::Sparse->new();	# Statistics calculator for means
+$meanStats->add_data(@means);
+my $stdevStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for stdevs
+$stdevStats->add_data(@stdevs);
+my $mean = $meanStats->mean();	# Mean of the sample of means
+my $stdev = $meanStats->standard_deviation(); # Stdev of the sample of means
+if (not defined $stdev) {
+    $stdev = 0;
+}
+my $meanStdev = $stdevStats->mean(); # Mean of the sample of stdevs
+if (defined REJECT_SAMPLE_MEAN->{$detType} and
+    $stdev != 0 and
+    $mean / $stdev > REJECT_SAMPLE_MEAN->{$detType}) {
+    print "Rejecting exposure based on bad mean: " . ($mean / $stdev) . " vs " .
+	REJECT_SAMPLE_MEAN->{$detType} . "\n";
+    $reject = 1;
+} elsif (defined REJECT_SAMPLE_STDEV->{$detType} and
+	 $meanStdev > REJECT_SAMPLE_STDEV->{$detType}) {
+    print "Rejecting exposure based on bad mean stdev: " . $meanStdev . " vs " .
+	REJECT_SAMPLE_STDEV->{$detType} . "\n";
     $reject = 1;
 }
 
-
 # Add the result into the database
 {
-    my $command = "dettool -addresidexp -det_id $detId -iter $iter -exp_id $expId " .
+    my $command = "$dettool -addresidexp -det_id $detId -iteration $iter -exp_id $expId " .
 	"-recip " . RECIPE() . " -b1_uri $jpeg1Name -b2_uri $jpeg2Name -bg $mean -bg_stdev $stdev " .
 	"-bg_mean_stdev $meanStdev";	# Command to run