Index: trunk/ippScripts/scripts/detrend_reject_imfile.pl
===================================================================
--- trunk/ippScripts/scripts/detrend_reject_imfile.pl	(revision 9453)
+++ trunk/ippScripts/scripts/detrend_reject_imfile.pl	(revision 9454)
@@ -41,4 +41,5 @@
 use constant RECIPE2 => 'PPIMAGE_J2'; # Recipe to use for ppImage to make JPEGs
 
+#### XXXXX these values must come from the config system, and may depend on filter!!!
 # The expected mean, as a function of detrend type
 use constant EXPECTED_MEAN => {
@@ -46,10 +47,11 @@
     'dark' => 0,		# Dark should be zero
     'shutter' => undef,		# Shutter could be anything (depends on exposure level)
-    'flat' => 0		# Flat could be anything (depends on exposure level)
+    'flat' => 0		        # Flat could be anything (depends on exposure level)
     };
 
 # Rejection threshold for the mean, in terms of the standard deviation
 # This measures how close it is to what's expected
-use constant REJECT_INDIVIDUAL_MEAN => {
+# XXX it is valid to reject on more than one criterion
+use constant REJECT_COMPONENT_MEAN => {
     'bias' => 0,		# Should be fairly flat; some CRs
     'dark' => 0,		# Lots of CRs
@@ -59,6 +61,6 @@
 
 # Rejection threshold for the standard deviation, in ADUs
-# This measures how much variation there is in each component
-use constant REJECT_INDIVIDUAL_STDEV => {
+# This measures how much variation there is in each imfile component
+use constant REJECT_COMPONENT_STDEV => {
     'bias' => 0,		# Should be fairly flat; some CRs
     'dark' => 0,		# Lots of CRs
@@ -77,6 +79,15 @@
 
 # Rejection threshold for the stdev of the sample, in ADUs
-# This measures how much variation there is across the components
+# This measures how much variation there is across the imfile components
 use constant REJECT_SAMPLE_STDEV => {
+    'bias' => 0,		# Should be little variation between chips
+    'dark' => 0,		# Could be some glow on some chips
+    'shutter' => undef,		# Can be significant structure
+    'flat' => 0		        # Could be features on some chips, but all should be about the same
+    };
+
+# Rejection threshold for the stdev of the sample, in ADUs
+# This measures how much variation there is across the imfile components
+use constant REJECT_SAMPLE_MEAN_STDEV => {
     'bias' => 0,		# Should be little variation between chips
     'dark' => 0,		# Could be some glow on some chips
@@ -151,8 +162,8 @@
 die "Unknown expected mean for detrend type $det_type\n"
     if not exists EXPECTED_MEAN->{$det_type};
-die "Unknown individual mean rejection level for detrend type $det_type\n"
-    if not exists REJECT_INDIVIDUAL_MEAN->{$det_type};
-die "Unknown individual stdev rejection level for detrend type $det_type\n"
-    if not exists REJECT_INDIVIDUAL_STDEV->{$det_type};
+die "Unknown component mean rejection level for detrend type $det_type\n"
+    if not exists REJECT_COMPONENT_MEAN->{$det_type};
+die "Unknown component stdev rejection level for detrend type $det_type\n"
+    if not exists REJECT_COMPONENT_STDEV->{$det_type};
 die "Unknown sample mean rejection level for detrend type $det_type\n"
     if not exists REJECT_SAMPLE_MEAN->{$det_type};
@@ -160,29 +171,31 @@
     if not exists REJECT_SAMPLE_STDEV->{$det_type};
 
-# Reject based on the individual stats
+# Reject based on the stats of the component imfile
 die "Number of means and number of stdevs differ!\n" if scalar @means != scalar @stdevs;
 for (my $i = 0; $i < scalar @means; $i++) {
     my $mean = $means[$i];	# Mean for this component
     $mean -= EXPECTED_MEAN->{$det_type} if defined EXPECTED_MEAN->{$det_type};
-    my $stdev = $stdevs[$i];	# Stdev for this component
-    my $not_reject;
-
-    if (defined REJECT_INDIVIDUAL_MEAN->{$det_type}
-	and $stdev > 0 and
-	$mean / $stdev > REJECT_INDIVIDUAL_MEAN->{$det_type}) {
-	print "Rejecting exposure based on bad individual mean for component $i: " .
-	    ($mean / $stdev) . " vs " . REJECT_INDIVIDUAL_MEAN->{$det_type} . "\n";
-	$not_reject = 1;
-	last;
-    } elsif (defined REJECT_INDIVIDUAL_STDEV->{$det_type} and
-	     $stdev > REJECT_INDIVIDUAL_STDEV->{$det_type}) {
-	print "Rejecting exposure based on bad individual stdev for component $i: " .
-	    $stdev . " vs " . REJECT_INDIVIDUAL_STDEV->{$det_type} . "\n";
-	$not_reject = 1;
-	last;
+    my $stdev = sqrt($stdevs[$i]);	# Stdev for this component
+
+    ## IT IS VALID to reject on more than one criterion
+    if (REJECT_COMPONENT_MEAN->{$det_type} && ($stdev > 0) ) {
+	if ($mean / $stdev > REJECT_COMPONENT_MEAN->{$det_type}) {
+	    print "Rejecting exposure based on bad component mean for component $i: " .
+		($mean / $stdev) . " vs " . REJECT_COMPONENT_MEAN->{$det_type} . "\n";
+	    $reject = 1;
+	    last;
+	}
+    } 
+    if (REJECT_COMPONENT_STDEV->{$det_type}) {
+	if ($stdev > REJECT_COMPONENT_STDEV->{$det_type}) {
+	    print "Rejecting exposure based on bad component stdev for component $i: " .
+		$stdev . " vs " . REJECT_COMPONENT_STDEV->{$det_type} . "\n";
+	    $reject = 1;
+	    last;
+	}
     }
 }
 
-# Reject based on the sample stats
+# calculate the imfile ensemble statistics
 my $meanStats = Statistics::Descriptive::Sparse->new();	# Statistics calculator for means
 $meanStats->add_data(@means);
@@ -190,22 +203,33 @@
 $stdevStats->add_data(@stdevs);
 my $mean = $meanStats->mean();	# Mean of the sample of means
-my $stdev = $meanStats->standard_deviation(); # Stdev of the sample of means
+my $meanStdev = $meanStats->standard_deviation(); # Stdev of the sample of means
 if (not defined $stdev) {
     $stdev = 0;
 }
-my $meanStdev = sqrt($stdevStats->mean()); # Root-Mean-Square of the sample of stdevs
-my $not_reject;
+my $stdev = sqrt($stdevStats->mean()); # Root-Mean-Square of the sample of stdevs
 print "calculating root-mean-square of the sample stdevs: $meanStdev\n";
-if (defined REJECT_SAMPLE_MEAN->{$det_type} and
-    $stdev != 0 and
-    $mean / $stdev > REJECT_SAMPLE_MEAN->{$det_type}) {
-    print "Rejecting exposure based on bad mean: " . ($mean / $stdev) . " vs " .
-	REJECT_SAMPLE_MEAN->{$det_type} . "\n";
-    $not_reject = 1;
-} elsif (defined REJECT_SAMPLE_STDEV->{$det_type} and
-	 $meanStdev > REJECT_SAMPLE_STDEV->{$det_type}) {
-    print "Rejecting exposure based on bad mean stdev: " . $meanStdev . " vs " .
-	REJECT_SAMPLE_STDEV->{$det_type} . "\n";
-    $not_reject = 1;
+
+## Reject based on the imfile ensemble stats
+# reject if the imfile ensemble 
+if (REJECT_SAMPLE_MEAN->{$det_type} && ($stdev > 0)) {
+    if ($mean / $stdev > REJECT_SAMPLE_MEAN->{$det_type}) {
+	print "Rejecting exposure based on bad mean: " . ($mean / $stdev) . " vs " .
+	    REJECT_SAMPLE_MEAN->{$det_type} . "\n";
+	$reject = 1;
+    } 
+}
+if (REJECT_SAMPLE_STDEV->{$det_type}) {
+    if ($stdev > REJECT_SAMPLE_STDEV->{$det_type}) {
+	print "Rejecting exposure based on bad mean stdev: " . $stdev . " vs " .
+	    REJECT_SAMPLE_STDEV->{$det_type} . "\n";
+	$reject = 1;
+    }
+}
+if (REJECT_SAMPLE_MEAN_STDEV->{$det_type}) {
+    if ($meanStdev > REJECT_SAMPLE_MEAN_STDEV->{$det_type}) {
+	print "Rejecting exposure based on bad mean stdev: " . $meanStdev . " vs " .
+	    REJECT_SAMPLE_MEAN_STDEV->{$det_type} . "\n";
+	$reject = 1;
+    }
 }
 
@@ -229,2 +253,33 @@
 
 __END__
+
+####
+
+## this function is not well though out.  it loads a collection of
+## values (background mean and stdev) for a collection of imfile
+## subcomponents.  it then measures the mean background, the
+## background stdev (rms of ensemble stdevs), and the background mean
+## stdev (rms of the means).  it then tries to reject based on the
+## following criteria:
+## (background / stdev > limit) for a single component
+##    ---> this will ACCEPT for a poor images (large stdev)
+## (stdev > limit)
+## (mean background / stdev > limit) same problem
+## (mean stdev > limit)
+
+### I would suggest the following:
+## 1) calculate the ensemble mean and stdev
+## 2) for each component, is the (back - mean) / mean stdev > limit?
+##    (ie, is it an outlier?)
+## 3) is the component stdev > limit? (absolute test)
+## 4) calculate the background stdev for the ensemble (excluding rejects)
+## 5) is the mean stdev > limit (poor images)
+## 6) calculate the ensemble stdev (rms) of remaining (excluding rejects)
+## 7) is the ensemble stdev > limit 
+
+## for flats, the value used for stdev should be the fractional stdev
+## (stdev/mean) and no rejection should be performed on the basis of
+## the mean value, but could still be done on the mean stdev
+
+## the same logical cuts above can be applied to components in an
+## imfile, imfiles in an exposure, and exposures in a stack