Changeset 10333

Timestamp:

Nov 30, 2006, 4:34:00 PM (20 years ago)

Author:

Paul Price

Message:

Updating rejection criteria.

Location:

trunk/ippScripts/scripts

Files:

• detrend_reject_exp.pl (modified) (8 diffs)
• detrend_reject_imfile.pl (modified) (2 diffs)

trunk/ippScripts/scripts/detrend_reject_exp.pl

@@ -58 +58 @@
 my @expTags;                    # Array of exposure IDs
 my @means;                      # Array of means
+my @stdevs;                     # Array of standard deviations
+my @meanStdevs;                 # Array of standard deviations of the mean
 my @variances;                  # Array of variances
 my @accept;                     # Array of accept flags
@@ -70 +72 @@
     push @expTags, $exposure->{exp_tag};
     push @means, $exposure->{bg};
+    push @stdevs, $exposure->{bg_stdev};
+    push @meanStdevs, $exposure->{bg_mean_stdev};
     push @variances, $exposure->{bg_stdev}**2;
     push @accept, $exposure->{accept};
@@ -76 +80 @@
 my $meanStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
 $meanStats->add_data(@means);
-my $varianceStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
-$varianceStats->add_data(@variances);
+my $stdevStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
+$stdevStats->add_data(@stdevs);
+my $meanStdevStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
+$meanStdevStats->add_data(@meanStdevs);
 
 $ipprc->define_camera($camera);
@@ -85 +91 @@
 my $reject_meanstdev = rejection_limit( 'ENSEMBLE.MEANSTDEV', $det_type, $filter );
 
-# rejections based on comparison with ensemble statistics
-## these should probably be 3-sigma clipped statistics themselves...
-my $mean = $meanStats->mean();
-my $meanStdev = $meanStats->standard_deviation();
-if (not defined $meanStdev) { $meanStdev = 0; }
-
-my $var = $varianceStats->mean();
-my $varStdev = $varianceStats->standard_deviation();
-
-print "Ensemble mean $mean +/- $meanStdev, stdev " . sqrt($var) . " +/- " . sqrt($varStdev) . "\n\n";
+print "Ensemble mean " . $meanStats->mean() . " +/- " . $meanStats->standard_deviation .
+    ", stdev " . $stdevStats->mean() . " +/- " . $stdevStats->standard_deviation() . "\n\n";
 
 # Go through again to do rejection, and update the database for each exposure
@@ -111 +109 @@
         goto UPDATE;
     }
-    if ($reject_mean && ($meanStdev > 0)) {
-        my $nSigma = abs($means[$i] - $mean) / $meanStdev;
+
+    if ($reject_mean and defined $meanStats->standard_deviation() ) {
+        my $nSigma = abs($means[$i] - $meanStats->mean() ) / $meanStats->standard_deviation();
         if ($nSigma > $reject_mean) {
             print "Rejecting $expTag based on outlier mean value: " .
-                "$means[$i] is $nSigma vs " . $reject_mean . "\n";
+                "$means[$i] --> $nSigma sigma vs " . $reject_mean . "\n";
             $reject = 1;
             goto UPDATE;
@@ -122 +121 @@
         print "no rejection for exposure mean\n";
     }
-    if ($reject_stdev && ($varStdev > 0)) {
-        my $nSigma = abs($variances[$i] - $var) / $varStdev;
+
+    if ($reject_stdev and defined $stdevStats->standard_deviation() > 0) {
+        my $nSigma = abs($stdevs[$i] - $stdevStats->mean() ) / $stdevStats->standard_deviation();
         if ($nSigma > $reject_stdev) {
             print "Rejecting $expTag based on outlier stdev: " .
-                sqrt($variances[$i]) . " is $nSigma vs " . $reject_stdev . "\n";
+                $stdevs[$i] . " --> $nSigma sigma vs " . $reject_stdev . "\n";
             $reject = 1;
             goto UPDATE;
@@ -132 +132 @@
     } else {
         print "no rejection for exposure stdev\n";
+    }
+    
+    if ($reject_meanstdev and defined $meanStdevStats->standard_deviation() > 0) {
+        my $nSigma = abs($meanStdevs[$i] - $meanStdevStats->mean() ) / $meanStdevStats->standard_deviation();
+        if ($nSigma > $reject_meanstdev) {
+            print "Rejecting $expTag based on outlier mean stdev: " .
+                $meanStdevs[$i] . " --> $nSigma sigma vs " . $reject_stdev . "\n";
+            $reject = 1;
+            goto UPDATE;
+        }
+    } else {
+        print "no rejection for exposure mean stdev\n";
     }
     
@@ -174 +186 @@
 # Put the result into the database
 unless ($no_update) {
+    my $varianceStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
+    $varianceStats->add_data(@variances);
+
     my $command = "$dettool -adddetrunsummary -det_id $det_id -iteration $iter" .
-        " -bg " . $mean . " -bg_stdev " . sqrt($var) .
-        " -bg_mean_stdev " . $meanStdev;
+        " -bg " . $meanStats->mean() . " -bg_stdev " . sqrt($varianceStats->mean()) .
+        " -bg_mean_stdev " . $meanStats->standard_deviation();
     $command .= " -accept" if $master;
     

trunk/ippScripts/scripts/detrend_reject_imfile.pl

@@ -125 +125 @@
 
     if ($reject_imfile_mean) {
-        if (abs($mean) > $reject_imfile_mean) {
+        if (abs($mean) > $reject_imfile_mean * $stdev) {
             print "Rejecting exposure based on bad imfile mean for imfile $i: " .
                 $mean . " vs " . $reject_imfile_mean . "\n";
@@ -164 +164 @@
 # reject if the exposure ensemble mean is deviant
 if ($reject_exp_mean) {
-    if (abs($mean) > $reject_exp_mean) {
+    if (abs($mean) > $reject_exp_mean * $stdev) {
         print "Rejecting exposure based on bad mean: " . ($mean / $stdev) . " vs " .
             $reject_exp_mean . "\n";