Index: /trunk/ippScripts/scripts/detrend.pl
===================================================================
--- /trunk/ippScripts/scripts/detrend.pl	(revision 10127)
+++ /trunk/ippScripts/scripts/detrend.pl	(revision 10127)
@@ -0,0 +1,273 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use IPC::Cmd qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use Data::Dumper;
+
+my $mdcParser = PS::IPP::Metadata::Config->new;	# Metadata config parser
+
+# Look for programs we need
+my $missing_tools;
+my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
+my $detrend_process_imfile = can_run('detrend_process_imfile.pl')  or (warn "Can't find detrend_process_imfile.pl" and $missing_tools = 1);
+my $detrend_process_exp = can_run('detrend_process_exp.pl') or (warn "Can't find detrend_process_exp.pl" and $missing_tools = 1);
+my $detrend_stack = can_run('detrend_stack.pl') or (warn "Can't find detrend_stack.pl" and $missing_tools = 1);
+my $detrend_norm_calc = can_run('detrend_norm_calc.pl') or (warn "Can't find detrend_norm_calc.pl" and $missing_tools = 1);
+my $detrend_norm_apply = can_run('detrend_norm_apply.pl') or (warn "Can't find detrend_norm_apply.pl" and $missing_tools = 1);
+my $detrend_norm_exp = can_run('detrend_norm_exp.pl') or (warn "Can't find detrend_norm_exp.pl" and $missing_tools = 1);
+my $detrend_resid = can_run('detrend_resid.pl') or (warn "Can't find detrend_resid.pl" and $missing_tools = 1);
+my $detrend_reject_imfile = can_run('detrend_reject_imfile.pl') or (warn "Can't find detrend_reject_imfile.pl" and $missing_tools = 1);
+my $detrend_reject_exp = can_run('detrend_reject_exp.pl') or (warn "Can't find detrend_reject_exp.pl" and $missing_tools = 1);
+die "Can't find required tools.\n" if $missing_tools;
+
+# Process raw imfiles
+{
+    my $command = "$dettool -toprocessedimfile";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get detrend raw list: $error_code\n" if not $success;
+
+    my @whole = split /\n/, join( '', @$stdout_buf );
+    my @single = ();
+
+    while ( scalar @whole > 0 ) {
+	my $value = shift @whole;
+	push @single, $value;
+	if ($value =~ /^\s*END\s*$/) {
+	    push @single, "\n";
+
+	    my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
+		die "Unable to parse output from dettool.\n";
+	    
+	    foreach my $item (@$list) {
+		my $det_id = $item->{det_id};
+		my $det_type = $item->{det_type};
+		my $exp_tag = $item->{exp_tag};
+		my $class = $item->{class};
+		my $class_id = $item->{class_id};
+		my $uri = $item->{uri};
+		my $camera = $item->{camera};
+		
+		my $command = "$detrend_process_imfile --det_id $det_id --exp_tag $exp_tag --class $class --class_id $class_id --det_type $det_type --input_uri $uri --camera $camera";
+		my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		    run( command => $command, verbose => 1 );
+		die "Unable to do raw imfile processing on $exp_tag $class_id: $error_code\n" if not $success;
+	    }
+
+	    @single = ();
+
+	}	
+    }
+}
+
+# Process raw exposures
+{
+    my $command = "$dettool -toprocessedexp";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get detrend raw list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $exp_tag = $item->{exp_tag};
+	my $camera = $item->{camera};
+	my $det_id = $item->{det_id};
+
+	my $command = "$detrend_process_exp --det_id $det_id --exp_tag $exp_tag --camera $camera";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to do raw exposure processing on $det_id $exp_tag: $error_code\n" if not $success;
+    }
+}
+
+# Stack
+{
+    my $command = "$dettool -tostacked";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get stack list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $det_id = $item->{det_id};
+	my $iteration = $item->{iteration};
+	my $class_id = $item->{class_id};
+	my $det_type = $item->{det_item};
+	my $camera = $item->{camera};
+
+	my $command = "$detrend_stack --det_id $det_id --iteration $iteration --class_id $class_id --det_type $det_type --camera $camera";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to stack detrend $det_id $iteration $class_id: $error_code\n" if not $success;
+    }
+}
+
+# Calculate normalisation
+{
+    my $command = "$dettool -tonormalizedstat";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get normalise calculation list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $det_id = $item->{det_id};
+	my $iteration = $item->{iteration};
+	my $det_type = $item->{det_item};
+
+	my $command = "$detrend_norm_calc --det_id $det_id --iteration $iteration --det_type $det_type";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to calculate normalisation for $det_id $iteration: $error_code\n" if not $success;
+    }
+}
+
+# Apply normalisation
+{
+    my $command = "$dettool -tonormalize";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get normalisation list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $det_id = $item->{det_id};
+	my $iteration = $item->{iteration};
+	my $det_type = $item->{det_item};
+	my $class_id = $item->{class_id};
+	my $value = $item->{norm};
+	my $uri = $item->{uri};
+	my $camera = $item->{camera};
+
+	my $command = "$detrend_norm_apply --det_id $det_id --iteration $iteration --class_id $class_id --value $value --input_uri $uri --camera $camera --det_type $det_type";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to apply normalisation for $det_id $iteration $class_id: $error_code\n" if not $success;
+    }
+}
+
+# Examine normalised exposure
+{
+    my $command = "$dettool -tonormalizedexp";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get normalised exposures list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $det_id = $item->{det_id};
+	my $iteration = $item->{iteration};
+	my $det_type = $item->{det_item};
+	my $class_id = $item->{class_id};
+	my $value = $item->{norm};
+	my $uri = $item->{uri};
+	my $camera = $item->{camera};
+
+	my $command = "$detrend_norm_exp --det_id $det_id --iteration $iteration --camera $camera --det_type $det_type";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to examine normalised exposure for $det_id $iteration: $error_code\n" if not $success;
+    }
+}
+
+# Get residuals
+{
+    my $command = "$dettool -toresidimfile";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get residual processing list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    my @whole = split /\n/, join( '', @$stdout_buf );
+    my @single = ();
+
+    while ( scalar @whole > 0 ) {
+	my $value = shift @whole;
+	push @single, $value;
+	if ($value =~ /^\s*END\s*$/) {
+	    push @single, "\n";
+
+	    my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
+		die "Unable to parse output from dettool.\n";
+	    
+	    foreach my $item (@$list) {
+		my $exp_tag = $item->{exp_tag};
+		my $camera = $item->{camera};
+		my $det_id = $item->{det_id};
+		my $iteration = $item->{iteration};
+		my $class_id = $item->{class_id};
+		my $det_type = $item->{det_type};
+		my $detrend = $item->{detrend};
+		my $uri = $item->{uri};
+
+		my $command = "$detrend_resid --det_id $det_id --iteration $iteration --exp_tag $exp_tag --class_id $class_id --det_type $det_type --detrend $detrend --camera $camera --input_uri $uri";
+		my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		    run( command => $command, verbose => 1 );
+		die "Unable to do residual processing on $exp_tag $class_id: $error_code\n" if not $success;
+	    }
+
+	    @single = ();
+
+	}
+    }
+}
+
+# Reject based on imfiles
+{
+    my $command = "$dettool -toresidexp";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get residual imfile list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $exp_tag = $item->{exp_tag};
+	my $camera = $item->{camera};
+	my $det_id = $item->{det_id};
+	my $iteration = $item->{iteration};
+	my $det_type = $item->{det_type};
+
+	my $command = "$detrend_reject_imfile --det_id $det_id --iteration $iteration --exp_tag $exp_tag --det_type $det_type --camera $camera";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to do imfile rejection on $exp_tag $det_id $iteration: $error_code\n" if not $success;
+    }
+}
+
+# Reject based on exposures
+{
+    my $command = "$dettool -residdetrun";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get residual exposure list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $camera = $item->{camera};
+	my $det_id = $item->{det_id};
+	my $iteration = $item->{iteration};
+	my $det_type = $item->{det_type};
+
+	my $command = "$detrend_reject_exp --det_id $det_id --iteration $iteration --det_type $det_type --camera $camera";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to do exposure rejection on $det_id $iteration: $error_code\n" if not $success;
+    }
+}
+
+
+__END__
+
+
Index: /trunk/ippScripts/scripts/inject.pl
===================================================================
--- /trunk/ippScripts/scripts/inject.pl	(revision 10127)
+++ /trunk/ippScripts/scripts/inject.pl	(revision 10127)
@@ -0,0 +1,119 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use IPC::Cmd qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use Data::Dumper;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+my ($camera,			# Camera used
+    $telescope,			# Telescope used
+    $exp_type,			# Type of exposure
+    $class,			# Prefix for class id
+    $num,			# Number of imfiles
+    $path,			# Path to use
+    );
+GetOptions(
+	   'camera|c=s' => \$camera,
+	   'telescope|t=s' => \$telescope,
+	   'exp_type|e=s' => \$exp_type,
+	   'class_id=s' => \$class,
+	   'num|n=i' => \$num,
+	   'path=s' => \$path,
+) or pod2usage( 2 );
+
+pod2usage(
+    -msg => "Required options: --camera --telescope --exp_type --num",
+    -exitval => 3,
+) unless defined $camera
+    and defined $telescope
+    and defined $exp_type
+    and defined $num
+    and defined $path;
+
+my $mdcParser = PS::IPP::Metadata::Config->new;	# Metadata config parser
+
+# Look for programs we need
+my $missing_tools;
+my $pxinject = can_run('pxinject')  or (warn "Can't find pxinject" and $missing_tools = 1);
+my $p0tool = can_run('p0tool') or (warn "Can't find p0tool" and $missing_tools = 1);
+my $phase0_imfile = can_run('phase0_imfile.pl') or (warn "Can't find phase0_imfile.pl" and $missing_tools = 1);
+my $phase0_exp = can_run('phase0_exp.pl') or (warn "Can't find phase0_exp.pl" and $missing_tools = 1);
+die "Can't find required tools.\n" if $missing_tools;
+
+
+# Inject new data into the database
+{
+    foreach my $exp ( @ARGV ) {
+	my $command = "$pxinject -newExp -exp_id $exp -inst $camera -telescope $telescope -exp_type $exp_type -imfiles $num"; # Command to run
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to inject $exp: $error_code\n" if not $success;
+
+	my @line = split(/\s+/, $$stdout_buf[0]); # The output line, containing the exposure tag
+	my $exp_tag = $line[2];	# The exposure tag
+	for (my $i = 0; $i < 36; $i++) {
+	    my $class_id;
+	    if (defined $class) {
+		$class_id = sprintf $class, $i; # Name for class_id
+	    } else {
+		$class_id = $i;
+	    }
+	    my $command = "$pxinject -newImfile -exp_tag $exp_tag -class chip -class_id $class_id -uri $path/$exp.$class_id.fits"; # Command to run
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run( command => $command, verbose => 1 );
+	    die "Unable to inject $exp $i: $error_code\n" if not $success;
+	}
+    }
+}
+
+# Phase 0 imfile processing
+{
+    my $command = "$p0tool -pendingimfile"; # Command to run
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get phase 0 imfile list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from p0tool.\n";
+
+    foreach my $item (@$list) {
+	my $exp_tag = $item->{exp_tag};
+	my $class = $item->{class};
+	my $class_id = $item->{class_id};
+	my $uri = $item->{uri};
+
+	my $command = "$phase0_imfile --exp_tag $exp_tag --class $class --class_id $class_id --uri $uri";
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to do phase 0 imfile processing on $exp_tag $class_id: $error_code\n" if not $success;
+    }
+}
+
+# Phase 0 exposure processing
+{
+    my $command = "$p0tool -pendingexp"; # Command to run
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get phase 0 exposure list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from p0tool.\n";
+  
+    foreach my $item (@$list) {
+	my $exp_tag = $item->{exp_tag};
+
+	my $command = "$phase0_exp --exp_tag $exp_tag";
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to do phase 0 exposure processing on $exp_tag: $error_code\n" if not $success;
+    }
+}
+
+
+__END__
+
+