Index: trunk/ippScripts/scripts/detrend_process_exp.pl
===================================================================
--- trunk/ippScripts/scripts/detrend_process_exp.pl	(revision 13760)
+++ trunk/ippScripts/scripts/detrend_process_exp.pl	(revision 14009)
@@ -17,5 +17,5 @@
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
-use Statistics::Descriptive;
+use File::Temp qw( tempfile );
 
 use PS::IPP::Config qw($PS_EXIT_SUCCESS
@@ -29,5 +29,4 @@
 		       );
 my $ipprc = PS::IPP::Config->new(); # IPP configuration
-use File::Temp qw( tempfile );
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -47,8 +46,7 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage(
-    -msg => "Required options: --det_id --det_type --exp_tag --camera",
-    -exitval => 3,
-) unless defined $det_id
+pod2usage( -msg => "Required options: --det_id --det_type --exp_tag --camera",
+	   -exitval => 3) 
+    unless defined $det_id
     and defined $det_type
     and defined $exp_tag
@@ -57,25 +55,25 @@
 $ipprc->define_camera($camera);
 
-# Recipes to use, as a function of the detrend type
-use constant RECIPES => {
-    'bin1' => {		# We're creating a master --- already processed the input
-	'bias'     => 'PPIMAGE_J1_IMAGE_B',	# Bias only
-	'dark'     => 'PPIMAGE_J1_IMAGE_B',	# Dark only
-	'shutter'  => 'PPIMAGE_J1_IMAGE_F',	# Shutter only
-	'flat'     => 'PPIMAGE_J1_IMAGE_F',	# Flat-field only
-	'domeflat' => 'PPIMAGE_J1_IMAGE_F',	# Flat-field only
-	'skyflat'  => 'PPIMAGE_J1_IMAGE_F',	# Flat-field only
-	'fringe'   => 'PPIMAGE_J1_IMAGE_R',	# Fringe only
-    },
-    'bin2' => {		# We're checking the master --- input is not already processed
-	'bias'     => 'PPIMAGE_J2_IMAGE_B',	# Bias only
-	'dark'     => 'PPIMAGE_J2_IMAGE_B',	# Dark only
-	'shutter'  => 'PPIMAGE_J2_IMAGE_F',	# Shutter only
-	'flat'     => 'PPIMAGE_J2_IMAGE_F',	# Flat-field only
-	'domeflat' => 'PPIMAGE_J2_IMAGE_F',	# Flat-field only
-	'skyflat'  => 'PPIMAGE_J2_IMAGE_F',	# Flat-field only
-	'fringe'   => 'PPIMAGE_J2_IMAGE_R',	# Fringe only
-    },
-};
+# Recipes to use based on reduction class
+$reduction = 'DETREND' unless defined $reduction;
+
+my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE' . uc($det_type); # Recipe to use
+&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
+
+my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE' . uc($det_type); # Recipe to use
+&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
+
+# values to extract from output metadata and the stats to calculate
+# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
+# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
+# XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
+my $STATS = 
+   [   
+       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
+       { name => "bg",             type => "mean",  flag => "-bg" },
+       { name => "bg",             type => "stdev", flag => "-bg_mean_stdev" },
+       { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
+       # { name => "bg_mean_stdev",  type => "rms",   flag => "-bg_mean_stdev" },
+   ];
 
 # Look for programs we need
@@ -87,6 +85,4 @@
     exit($PS_EXIT_CONFIG_ERROR); 
 }
-
-my $mdcParser = PS::IPP::Metadata::Config->new;	# Parser for metadata config files
 
 # Get list of component files
@@ -101,34 +97,18 @@
 	&my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $exp_tag, $error_code);
     }
+
+    # convert stdout to a metadata
+    my $mdcParser = PS::IPP::Metadata::Config->new;	# Parser for metadata config files
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
 	&my_die("Unable to parse metadata config doc", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR);
+
+    # parse the file info in the metadata
     $files = parse_md_list($metadata) or
 	&my_die("Unable to parse metadata list", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR);
-}
-
-# Gather the statistics
-my ($bg, $bg_stdev, $bg_mean_stdev); # The statistics triplet
-{
-    my @backgrounds;		# Array of backgrounds in each component
-    my @variances;    # Array of variances for each component
-    foreach my $file (@$files) {
-	&my_die("Unable to find class id", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{class_id};
-	my $class_id = $file->{class_id};
-	&my_die("Unable to find bg for class_id=$class_id", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{bg};
-	&my_die("Unable to find bg_mean_stdev for class_id=$class_id", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{bg_mean_stdev};
-	push @backgrounds, $file->{bg};
-	push @variances, $file->{bg_stdev}**2;
-    }
-
-    {
-	my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
-	$stats->add_data(@backgrounds);
-	$bg = ($stats->mean() or 'NAN');
-	$bg_mean_stdev = ($stats->standard_deviation() or 'NAN');
-    }
-    {
-	my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
-	$stats->add_data(@variances);
-	$bg_stdev = (sqrt( $stats->mean() ) or 'NAN');
+
+    # parse the stats in the metadata
+    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
+    unless ($stats->parse($metadata)) {
+	&my_die("Unable to find all values in statistics output.\n", $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
     }
 }
@@ -137,11 +117,7 @@
 my ($list1File, $list1Name) = tempfile( "$exp_tag.detproc.$det_id.b1.list.XXXX", UNLINK => 1 );
 my ($list2File, $list2Name) = tempfile( "$exp_tag.detproc.$det_id.b2.list.XXXX", UNLINK => 1 );
-my @means;			# Array of means
-my @stdevs;			# Array of stdevs
 foreach my $file (@$files) {
     print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n");
     print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n");
-    push @means, $file->{bg};
-    push @stdevs, $file->{bg_stdev};
 }
 close $list1File;
@@ -153,11 +129,4 @@
 my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outputRoot); # Binned JPEG #1
 my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2
-
-# Recipes to use in processing
-my $recipe1 = RECIPES->{"bin1"}->{lc($det_type)};
-&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
-
-my $recipe2 = RECIPES->{"bin2"}->{lc($det_type)};
-&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
 
 unless ($no_op) {
@@ -187,9 +156,20 @@
 }
 
+# Command to update the database
+my $command = "$dettool -addprocessedexp";
+$command .= " -det_id $det_id";
+$command .= " -exp_tag $exp_tag";
+$command .= " -recip $recipe1,$recipe2 -path_base $outputRoot";
+$command .= " -dbname $dbname" if defined $dbname;
+
+# add in the elements from the selected stats above
+foreach my $entry (@$STATS) {
+    my $value = $entry->{value};
+    my $flag = $entry->{flag};
+    $command .= " $flag $value";
+}
+
+# Add the processed file to the database
 unless ($no_update) {
-    my $command = "$dettool -addprocessedexp -det_id $det_id -exp_tag $exp_tag";
-    $command .= " -recip $recipe1,$recipe2 -path_base $outputRoot";
-    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
-    $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 	run(command => $command, verbose => 1);
@@ -199,6 +179,7 @@
 	exit($error_code);
     }
-}
-
+} else {
+    print "skipping command: $command\n";
+}
 
 sub my_die
@@ -211,5 +192,8 @@
     carp($msg);
     if ($det_id and $exp_tag and not $no_update) {
-	my $command = "$dettool -addprocessedexp -det_id $det_id -exp_tag $exp_tag -code $exit_code";
+	my $command = "$dettool -addprocessedexp";
+	$command .= " -det_id $det_id";
+	$command .= " -exp_tag $exp_tag";
+	$command .= " -code $exit_code";
 	$command .= " -dbname $dbname" if defined $dbname;
 ###        system ($command);