Index: trunk/ippScripts/scripts/detrend_reject_imfile.pl
===================================================================
--- trunk/ippScripts/scripts/detrend_reject_imfile.pl	(revision 13698)
+++ trunk/ippScripts/scripts/detrend_reject_imfile.pl	(revision 13722)
@@ -1,3 +1,9 @@
 #!/usr/bin/env perl
+
+# this program has two jobs:
+# 1: for the given exp_tag, generate the binned & mosaiced JPEG images
+# 2: examine the collection of per-imfile statistics and reject.  At the moment, 
+#    this program (and the database) only allows rejection at the exposure level,
+#    not at the imfile level. 
 
 use warnings;
@@ -13,11 +19,10 @@
 $VERSION = '0.01';
 
-use IPC::Cmd 0.36 qw( can_run run );
-use PS::IPP::Metadata::Config;
-use PS::IPP::Metadata::List qw( parse_md_list );
-use Statistics::Descriptive;
-use File::Temp qw( tempfile );
-
-use PS::IPP::Config qw($PS_EXIT_SUCCESS
+use IPC::Cmd 0.36 qw( can_run run );             # tools to run UNIX programs with control over I/O
+use PS::IPP::Metadata::Config;                   # tools to parse the psMetadataConfig files
+use PS::IPP::Metadata::List qw( parse_md_list ); # ?
+use Statistics::Descriptive;                     # tools for calculating basic statistical quantities
+use File::Temp qw( tempfile );                   # tools to construct temp files
+use PS::IPP::Config qw($PS_EXIT_SUCCESS          
 		       $PS_EXIT_UNKNOWN_ERROR
 		       $PS_EXIT_SYS_ERROR
@@ -27,10 +32,11 @@
 		       $PS_EXIT_TIMEOUT_ERROR
 		       caturi
-		       );
+		       );                        # tools to parse the IPP configuration information
 my $ipprc = PS::IPP::Config->new(); # IPP configuration
 
-use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
-use Pod::Usage qw( pod2usage );
-
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); # option parsing
+use Pod::Usage qw( pod2usage ); 
+
+# parse the command-line options
 my ($det_id, $iter, $exp_tag, $det_type, $camera, $filter, $reject, $dbname, $workdir, $no_update, $no_op);
 GetOptions(
@@ -58,4 +64,5 @@
     and defined $camera;
 
+# load IPP config information for the specified camera
 $ipprc->define_camera($camera);
 
@@ -91,10 +98,15 @@
 }
 
-my $mdcParser = PS::IPP::Metadata::Config->new;	# Parser for metadata config files
+# Parser for psMetadataConfig files (used for output from dettool)
+my $mdcParser = PS::IPP::Metadata::Config->new;	
 
 # Get list of imfile files
 my $files;			# Array of imfile files
 {
-    my $command = "$dettool -residimfile -det_id $det_id -iteration $iter -exp_tag $exp_tag"; # Command to run
+    # dettool command to select imfile data for this exp_tag
+    my $command = "$dettool -residimfile";
+    $command .= " -det_id $det_id";
+    $command .= " -iteration $iter";
+    $command .= " -exp_tag $exp_tag";
     $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -105,5 +117,4 @@
 	exit($error_code);
     }
-
     # XXX report an error message if stdout_buf is empty
 
@@ -114,25 +125,4 @@
 	&my_die("Unable to parse metadata list", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR);
 }
-
-# XXX in debug mode, unlink = 0
-my ($list1File, $list1Name) = tempfile( "$exp_tag.detresid.$det_id.$iter.b1.list.XXXX", UNLINK => 1 );
-my ($list2File, $list2Name) = tempfile( "$exp_tag.detresid.$det_id.$iter.b2.list.XXXX", UNLINK => 1 );
-my @means;			# Array of means
-my @variances;			# Array of variances
-my @binVariances;		# Array of binned variances
-my @meanStdevs;			# Array of mean stdevs
-my @names;			# Array of names (class_id)
-foreach my $file (@$files) {
-    print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n");
-    print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n");
-    push @means, $file->{bg};
-    push @meanStdevs, $file->{bg_mean_stdev};
-    ## calculate the root-mean-square of the bd_stdevs
-    push @variances, $file->{bg_stdev}*$file->{bg_stdev};
-    push @binVariances, $file->{bin_stdev}*$file->{bin_stdev};
-    push @names, $file->{class_id};
-}
-close $list1File;
-close $list2File;
 
 # Output products
@@ -146,6 +136,18 @@
 unless ($no_op) {
     # XXX this will fail if the file exists (because of Nebulous rules)
-    $logFile = $ipprc->file_create_open( $logName );
-}
+    $logFile = $ipprc->file_create_append( $logName );
+}
+
+#### build the JPEG images ####
+
+# XXX in debug mode, unlink = 0
+my ($list1File, $list1Name) = tempfile( "$exp_tag.detresid.$det_id.$iter.b1.list.XXXX", UNLINK => 1 );
+my ($list2File, $list2Name) = tempfile( "$exp_tag.detresid.$det_id.$iter.b2.list.XXXX", UNLINK => 1 );
+foreach my $file (@$files) {
+    print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n");
+    print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n");
+}
+close $list1File;
+close $list2File;
 
 # Recipes to use in processing
@@ -183,16 +185,44 @@
 }
 
-my $expected = rejection_limit( 'EXPECTED', $det_type, $filter ); # Expected mean
-# Rejection thresholds
+#### measure stats and reject if needed ####
+
+# Rejection thresholds & related data
+my $expected                = rejection_limit( 'EXPECTED',         $det_type, $filter );
 my $reject_imfile_mean      = rejection_limit( 'IMFILE.MEAN',      $det_type, $filter );
+my $reject_imfile_flux      = rejection_limit( 'IMFILE.FLUX',      $det_type, $filter );
 my $reject_imfile_stdev     = rejection_limit( 'IMFILE.STDEV',     $det_type, $filter );
 my $reject_imfile_meanstdev = rejection_limit( 'IMFILE.MEANSTDEV', $det_type, $filter );
 my $reject_imfile_sn        = rejection_limit( 'IMFILE.SN',        $det_type, $filter );
+my $reject_imfile_bin_stdev = rejection_limit( 'IMFILE.BIN.STDEV', $det_type, $filter );
 my $reject_imfile_bin_sn    = rejection_limit( 'IMFILE.BIN.SN',    $det_type, $filter );
 my $reject_exp_mean         = rejection_limit( 'EXP.MEAN',         $det_type, $filter );
+my $reject_exp_flux         = rejection_limit( 'EXP.FLUX',         $det_type, $filter );
 my $reject_exp_stdev        = rejection_limit( 'EXP.STDEV',        $det_type, $filter );
 my $reject_exp_meanstdev    = rejection_limit( 'EXP.MEANSTDEV',    $det_type, $filter );
 my $reject_exp_sn           = rejection_limit( 'EXP.SN',           $det_type, $filter );
+my $reject_exp_bin_stdev    = rejection_limit( 'EXP.BIN.STDEV',    $det_type, $filter );
 my $reject_exp_bin_sn       = rejection_limit( 'EXP.BIN.SN',       $det_type, $filter );
+
+# storage variables
+my @means;			# Array of means
+my @fluxes;			# Array of means / exptimes
+my @variances;			# Array of variances
+my @binVariances;		# Array of binned variances
+my @meanStdevs;			# Array of mean stdevs
+my @names;			# Array of names (class_id)
+
+# load the arrays from the imfile output table
+foreach my $file (@$files) {
+    push @means,      	$file->{bg};                           # mean background counts
+    if ($file->{exp_time} > 0.0) {
+	push @fluxes,     	$file->{bg} / $file->{exp_time};       # mean background counts / sec
+    } else {
+	push @fluxes,     	$file->{bg};
+    }
+    push @meanStdevs, 	$file->{bg_mean_stdev};                # stdev of the mean counts (for imfile components)
+    push @variances,  	$file->{bg_stdev}*$file->{bg_stdev};   # total variance from all imfile components
+    push @binVariances, $file->{bin_stdev}*$file->{bin_stdev}; # total variance of binned images over all imfile components
+    push @names,        $file->{class_id};                     # name of the component
+}
 
 # Reject based on the stats of the imfiles
@@ -200,4 +230,5 @@
 &my_die("Number of means and number of variances differ!", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR) unless scalar @means == scalar @variances;
 for (my $i = 0; $i < scalar @means; $i++) {
+    my $flux = $fluxes[$i];	# Flux for this imfile
     my $mean = $means[$i];	# Mean for this imfile
     $mean -= $expected;
@@ -208,4 +239,9 @@
     last if $no_op;
 
+    # reject exposure if, for any imfiles, the mean residual counts
+    # deviate from the expected value by more than N sigma, (sigma =
+    # total pixel variance).  this test is sensible for images which
+    # should have a predictable nominal residual mean count value (eg,
+    # 0.0 for a bias).   in general, use with ADDITIVE components
     if ($reject_imfile_mean > 0) {
 	if (abs($mean) > $reject_imfile_mean * $stdev) {
@@ -215,8 +251,28 @@
 	    print $logFile "Imfile mean for $name meets requirements: ";
 	}
-	print $logFile "$mean vs $reject_imfile_mean\n";
+	print $logFile "$mean vs $reject_imfile_mean * $stdev\n";
     }  else {
 	print $logFile "No rejection on imfile mean for $name\n";
     }
+
+    # reject exposure if, for any imfiles, the mean residual flux
+    # deviates from the expected value by more than N sigma, (sigma =
+    # total pixel variance).  this test is sensible for images which
+    # should have a predictable nominal residual flux value (eg, 0.0
+    # for a bias).  in general, use with ADDITIVE components
+    if ($reject_imfile_flux > 0) {
+	if (abs($flux) > $reject_imfile_flux) {
+	    print $logFile "Rejecting exposure based on bad imfile flux for $name: ";
+	    $reject = 1;
+	} else {
+	    print $logFile "Imfile flux for $name meets requirements: ";
+	}
+	print $logFile "$flux vs $reject_imfile_flux\n";
+    }  else {
+	print $logFile "No rejection on imfile flux for $name\n";
+    }
+
+    # reject exposure if, for any imfiles, the total pixel variance is
+    # larger than the limit
     if ($reject_imfile_stdev > 0) {
 	if ($stdev > $reject_imfile_stdev) {
@@ -231,4 +287,7 @@
 	print $logFile "No rejection on imfile stdev for $name\n";
     }
+
+    # reject exposure if, for any imfiles, the variance of the imfile
+    # component means is larger than the limit
     if ($reject_imfile_meanstdev > 0) {
 	if ($meanStdevs[$i] > $reject_imfile_meanstdev) {
@@ -242,4 +301,9 @@
 	print $logFile "No rejection on imfile mean stdev for $name\n";
     }
+
+    # reject exposure if, for any imfiles, the signal-to-noise (ie,
+    # the mean counts / total pixel variance) of the imfile component
+    # means are less than the limit.  this test is sensible for images
+    # which have finite residual flux such as a flat-field image.  
     if ($reject_imfile_sn > 0) {
 	if ($mean < $stdev * $reject_imfile_sn) {
@@ -253,4 +317,20 @@
 	print $logFile "No rejection on imfile S/N for $name\n";
     }
+
+    # reject exposure if, for any imfiles, the signal-to-noise of the
+    # imfile component means, based on the stdev of the binned image
+    # is less than the limit.  this test is sensible for images which
+    # have finite residual flux such as a flat-field image.
+    if ($reject_imfile_bin_stdev > 0) {
+	if ($binStdev > $reject_imfile_bin_stdev) {
+	    print $logFile "Rejecting exposure based on bad imfile binned stdev for $name: ";
+	    $reject = 1;
+	} else {
+	    print $logFile "Imfile binned stdev for $name meets requirements: ";
+	}
+	print $logFile "$binStdev vs $reject_imfile_bin_sn\n";
+    } else {
+	print $logFile "No rejection on imfile binned stdev for $name\n";
+    }
     if ($reject_imfile_bin_sn > 0) {
 	if ($mean < $binStdev * $reject_imfile_bin_sn) {
@@ -269,4 +349,6 @@
 my $meanStats = Statistics::Descriptive::Sparse->new();	# Statistics calculator for means
 $meanStats->add_data(@means);
+my $fluxStats = Statistics::Descriptive::Sparse->new();	# Statistics calculator for means
+$fluxStats->add_data(@fluxes);
 my $varianceStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for variances
 $varianceStats->add_data(@variances);
@@ -275,4 +357,5 @@
 
 my $mean = $meanStats->mean();	# Mean of the imfile means
+my $flux = $fluxStats->mean();	# Mean of the imfile means
 my $meanStdev = $meanStats->standard_deviation(); # Stdev of the imfile means
 if (not defined $meanStdev) {
@@ -290,16 +373,40 @@
     print $logFile "Exposure mean $mean, stdev $stdev, mean stdev $meanStdev, exp s/n: $exp_sn\n";
 
+    # reject exposure if the ensemble mean residual counts deviate
+    # from the expected value by more than N sigma, (sigma = total
+    # pixel variance).  this test is sensible for images which should
+    # have a predictable nominal residual mean count value (eg, 0.0
+    # for a bias).  in general, use with ADDITIVE components
     if ($reject_exp_mean > 0) {
 	if (abs($mean) > $reject_exp_mean * $stdev) {
-	    print $logFile "Rejecting exposure based on bad mean: ";
-	    $reject = 1;
-	} else {
-	    print $logFile "Exposure mean meets requirements: ";
-	}
-	print $logFile "mean: $mean, stdev: $stdev (limit is: % $reject_exp_mean\n";
+	    print $logFile "Rejecting exposure based on bad mean counts: ";
+	    $reject = 1;
+	} else {
+	    print $logFile "Exposure mean counts meets requirements: ";
+	}
+	print $logFile "mean: $mean, stdev: $stdev (limit is: $reject_exp_mean\n";
     } else {
 	print $logFile "No rejection for exp mean\n";
     }
-    # reject if the exposure ensemble stdev is deviant
+
+    # reject exposure if, for any imfiles, the mean residual flux
+    # deviates from the expected value by more than N sigma, (sigma =
+    # total pixel variance).  this test is sensible for images which
+    # should have a predictable nominal residual flux value (eg, 0.0
+    # for a bias).  in general, use with ADDITIVE components
+    if ($reject_exp_flux > 0) {
+	if (abs($flux) > $reject_exp_flux * $stdev) {
+	    print $logFile "Rejecting exposure based on bad mean flux: ";
+	    $reject = 1;
+	} else {
+	    print $logFile "Exposure mean flux meets requirements: ";
+	}
+	print $logFile "flux: $flux, stdev: $stdev (limit is: $reject_exp_flux\n";
+    } else {
+	print $logFile "No rejection for exp mean\n";
+    }
+
+    # reject exposure if the total pixel variance is larger than the
+    # limit
     if ($reject_exp_stdev > 0) {
 	if ($stdev > $reject_exp_stdev) {
@@ -313,5 +420,7 @@
 	print $logFile "No rejection for exp stdev\n";
     }
-    # reject if the exposure ensemble mean stdev is deviant
+
+    # reject exposure if the variance of the imfile means is larger
+    # than the limit
     if ($reject_exp_meanstdev > 0) {
 	if ($meanStdev > $reject_exp_meanstdev) {
@@ -325,4 +434,5 @@
 	print $logFile "No rejection for exp mean stdev\n";
     }
+
     # reject if the signal-to-noise is insufficient
     if ($reject_exp_sn > 0) {
@@ -336,4 +446,16 @@
     } else {
 	print $logFile "No rejection for exp S/N\n";
+    }
+    # reject if the exposure ensemble stdev is deviant
+    if ($reject_exp_bin_stdev > 0) {
+	if ($binStdev > $reject_exp_bin_stdev) {
+	    print $logFile "Rejecting exposure based on bad binned stdev: ";
+	    $reject = 1;
+	} else {
+	    print $logFile "Exposure binned stdev meets requirements: ";
+	}
+	print $logFile "$stdev vs $reject_exp_bin_stdev\n";
+    } else {
+	print $logFile "No rejection for exp binned stdev\n";
     }
     # reject if the signal-to-noise is insufficient