Index: trunk/ippScripts/scripts/detrend.pl
===================================================================
--- trunk/ippScripts/scripts/detrend.pl	(revision 11408)
+++ 	(revision )
@@ -1,274 +1,0 @@
-#!/usr/bin/env perl
-
-use warnings;
-use strict;
-
-use IPC::Cmd qw( can_run run );
-use PS::IPP::Metadata::Config;
-use PS::IPP::Metadata::List qw( parse_md_list );
-use Data::Dumper;
-
-my $mdcParser = PS::IPP::Metadata::Config->new;	# Metadata config parser
-
-# Look for programs we need
-my $missing_tools;
-my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
-my $detrend_process_imfile = can_run('detrend_process_imfile.pl')  or (warn "Can't find detrend_process_imfile.pl" and $missing_tools = 1);
-my $detrend_process_exp = can_run('detrend_process_exp.pl') or (warn "Can't find detrend_process_exp.pl" and $missing_tools = 1);
-my $detrend_stack = can_run('detrend_stack.pl') or (warn "Can't find detrend_stack.pl" and $missing_tools = 1);
-my $detrend_norm_calc = can_run('detrend_norm_calc.pl') or (warn "Can't find detrend_norm_calc.pl" and $missing_tools = 1);
-my $detrend_norm_apply = can_run('detrend_norm_apply.pl') or (warn "Can't find detrend_norm_apply.pl" and $missing_tools = 1);
-my $detrend_norm_exp = can_run('detrend_norm_exp.pl') or (warn "Can't find detrend_norm_exp.pl" and $missing_tools = 1);
-my $detrend_resid = can_run('detrend_resid.pl') or (warn "Can't find detrend_resid.pl" and $missing_tools = 1);
-my $detrend_reject_imfile = can_run('detrend_reject_imfile.pl') or (warn "Can't find detrend_reject_imfile.pl" and $missing_tools = 1);
-my $detrend_reject_exp = can_run('detrend_reject_exp.pl') or (warn "Can't find detrend_reject_exp.pl" and $missing_tools = 1);
-die "Can't find required tools.\n" if $missing_tools;
-
-# Process raw imfiles
-{
-    my $command = "$dettool -toprocessedimfile";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run( command => $command, verbose => 1 );
-    die "Unable to get detrend raw list: $error_code\n" if not $success;
-
-    my @whole = split /\n/, join( '', @$stdout_buf );
-    my @single = ();
-
-    while ( scalar @whole > 0 ) {
-	my $value = shift @whole;
-	push @single, $value;
-	if ($value =~ /^\s*END\s*$/) {
-	    push @single, "\n";
-
-	    my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
-		die "Unable to parse output from dettool.\n";
-	    
-	    foreach my $item (@$list) {
-		my $det_id = $item->{det_id};
-		my $det_type = $item->{det_type};
-		my $exp_tag = $item->{exp_tag};
-		my $class = $item->{class};
-		my $class_id = $item->{class_id};
-		my $uri = $item->{uri};
-		my $camera = $item->{camera};
-		
-		my $command = "$detrend_process_imfile --det_id $det_id --exp_tag $exp_tag --class $class --class_id $class_id --det_type $det_type --input_uri $uri --camera $camera";
-		my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		    run( command => $command, verbose => 1 );
-		die "Unable to do raw imfile processing on $exp_tag $class_id: $error_code\n" if not $success;
-	    }
-
-	    @single = ();
-
-	}	
-    }
-}
-
-# Process raw exposures
-{
-    my $command = "$dettool -toprocessedexp";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run( command => $command, verbose => 1 );
-    die "Unable to get detrend raw list: $error_code\n" if not $success;
-    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
-	die "Unable to parse output from dettool.\n";
-
-    foreach my $item (@$list) {
-	my $exp_tag = $item->{exp_tag};
-	my $camera = $item->{camera};
-	my $det_id = $item->{det_id};
-
-	my $command = "$detrend_process_exp --det_id $det_id --exp_tag $exp_tag --camera $camera";
-    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run( command => $command, verbose => 1 );
-	die "Unable to do raw exposure processing on $det_id $exp_tag: $error_code\n" if not $success;
-    }
-}
-
-# Stack
-{
-    my $command = "$dettool -tostacked";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run( command => $command, verbose => 1 );
-    die "Unable to get stack list: $error_code\n" if not $success;
-    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
-	die "Unable to parse output from dettool.\n";
-
-    foreach my $item (@$list) {
-	my $det_id = $item->{det_id};
-	my $iteration = $item->{iteration};
-	my $class_id = $item->{class_id};
-	my $det_type = $item->{det_type};
-	my $camera = $item->{camera};
-
-	my $command = "$detrend_stack --det_id $det_id --iteration $iteration --class_id $class_id --det_type $det_type --camera $camera";
-    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run( command => $command, verbose => 1 );
-	die "Unable to stack detrend $det_id $iteration $class_id: $error_code\n" if not $success;
-    }
-}
-
-# Calculate normalisation
-{
-    my $command = "$dettool -tonormalizedstat";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run( command => $command, verbose => 1 );
-    die "Unable to get normalise calculation list: $error_code\n" if not $success;
-    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
-	die "Unable to parse output from dettool.\n";
-
-    foreach my $item (@$list) {
-	my $det_id = $item->{det_id};
-	my $iteration = $item->{iteration};
-	my $det_type = $item->{det_type};
-
-	my $command = "$detrend_norm_calc --det_id $det_id --iteration $iteration --det_type $det_type";
-    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run( command => $command, verbose => 1 );
-	die "Unable to calculate normalisation for $det_id $iteration: $error_code\n" if not $success;
-    }
-}
-
-# Apply normalisation
-{
-    my $command = "$dettool -tonormalize";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run( command => $command, verbose => 1 );
-    die "Unable to get normalisation list: $error_code\n" if not $success;
-    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
-	die "Unable to parse output from dettool.\n";
-
-    foreach my $item (@$list) {
-	my $det_id = $item->{det_id};
-	my $iteration = $item->{iteration};
-	my $det_type = $item->{det_type};
-	my $class_id = $item->{class_id};
-	my $value = $item->{norm};
-	my $uri = $item->{uri};
-	my $camera = $item->{camera};
-
-	my $command = "$detrend_norm_apply --det_id $det_id --iteration $iteration --class_id $class_id --value $value --input_uri $uri --camera $camera --det_type $det_type";
-    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run( command => $command, verbose => 1 );
-	die "Unable to apply normalisation for $det_id $iteration $class_id: $error_code\n" if not $success;
-    }
-}
-
-# Examine normalised exposure
-{
-    my $command = "$dettool -tonormalizedexp";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run( command => $command, verbose => 1 );
-    die "Unable to get normalised exposures list: $error_code\n" if not $success;
-    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
-	die "Unable to parse output from dettool.\n";
-
-    foreach my $item (@$list) {
-	my $det_id = $item->{det_id};
-	my $iteration = $item->{iteration};
-	my $det_type = $item->{det_type};
-	my $class_id = $item->{class_id};
-	my $value = $item->{norm};
-	my $uri = $item->{uri};
-	my $camera = $item->{camera};
-
-	my $command = "$detrend_norm_exp --det_id $det_id --iteration $iteration --camera $camera --det_type $det_type";
-    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run( command => $command, verbose => 1 );
-	die "Unable to examine normalised exposure for $det_id $iteration: $error_code\n" if not $success;
-    }
-}
-
-# Get residuals
-{
-    my $command = "$dettool -toresidimfile";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run( command => $command, verbose => 1 );
-    die "Unable to get residual processing list: $error_code\n" if not $success;
-
-    my @whole = split /\n/, join( '', @$stdout_buf );
-    my @single = ();
-
-    while ( scalar @whole > 0 ) {
-	my $value = shift @whole;
-	push @single, $value;
-	if ($value =~ /^\s*END\s*$/) {
-	    push @single, "\n";
-
-	    my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
-		die "Unable to parse output from dettool.\n";
-	    
-	    foreach my $item (@$list) {
-		my $exp_tag = $item->{exp_tag};
-		my $camera = $item->{camera};
-		my $det_id = $item->{det_id};
-		my $iteration = $item->{iteration};
-		my $class_id = $item->{class_id};
-		my $det_type = $item->{det_type};
-		my $detrend = $item->{det_uri};
-		my $uri = $item->{uri};
-		my $mode = $item->{mode};
-
-		my $command = "$detrend_resid --det_id $det_id --iteration $iteration --exp_tag $exp_tag --class_id $class_id --det_type $det_type --camera $camera --input_uri $uri --mode $mode";
-		$command .= " --detrend $detrend" if defined $detrend;
-
-		my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		    run( command => $command, verbose => 1 );
-		die "Unable to do residual processing on $exp_tag $class_id: $error_code\n" if not $success;
-	    }
-
-	    @single = ();
-
-	}
-    }
-}
-
-# Reject based on imfiles
-{
-    my $command = "$dettool -toresidexp";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run( command => $command, verbose => 1 );
-    die "Unable to get residual imfile list: $error_code\n" if not $success;
-    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
-	die "Unable to parse output from dettool.\n";
-
-    foreach my $item (@$list) {
-	my $exp_tag = $item->{exp_tag};
-	my $camera = $item->{camera};
-	my $det_id = $item->{det_id};
-	my $iteration = $item->{iteration};
-	my $det_type = $item->{det_type};
-
-	my $command = "$detrend_reject_imfile --det_id $det_id --iteration $iteration --exp_tag $exp_tag --det_type $det_type --camera $camera";
-    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run( command => $command, verbose => 1 );
-	die "Unable to do imfile rejection on $exp_tag $det_id $iteration: $error_code\n" if not $success;
-    }
-}
-
-# Reject based on exposures
-{
-    my $command = "$dettool -residdetrun";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run( command => $command, verbose => 1 );
-    die "Unable to get residual exposure list: $error_code\n" if not $success;
-    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
-	die "Unable to parse output from dettool.\n";
-
-    foreach my $item (@$list) {
-	my $camera = $item->{camera};
-	my $det_id = $item->{det_id};
-	my $iteration = $item->{iteration};
-	my $det_type = $item->{det_type};
-
-	my $command = "$detrend_reject_exp --det_id $det_id --iteration $iteration --det_type $det_type --camera $camera";
-    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run( command => $command, verbose => 1 );
-	die "Unable to do exposure rejection on $det_id $iteration: $error_code\n" if not $success;
-    }
-}
-
-
-__END__
-
-
Index: trunk/ippScripts/scripts/inject.pl
===================================================================
--- trunk/ippScripts/scripts/inject.pl	(revision 11408)
+++ 	(revision )
@@ -1,97 +1,0 @@
-#!/usr/bin/env perl
-
-# this program injects a list of single-file exposures into the db, taking the
-# filename (without .fits) as the exp_tag.
-
-use warnings;
-use strict;
-
-use IPC::Cmd qw( can_run run );
-use PS::IPP::Metadata::Config;
-use PS::IPP::Metadata::List qw( parse_md_list );
-use File::Spec;
-use File::Find::Rule;
-use PS::IPP::Config;
-my $ipprc = PS::IPP::Config->new();
-my $mdcParser = PS::IPP::Metadata::Config->new;	# Parser for metadata config files
-
-use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
-
-# Parse the command-line arguments
-my ($workdir, $dbname);
-GetOptions(
-    'workdir|w=s'   => \$workdir,  # working directory for output files
-    'dbname|d=s'    => \$dbname    # Database name
-) or pod2usage( 2 );
-
-pod2usage( -msg => "Usage: $0 [--workdir path] [--dbname dbname] (files)", -exitval => 2 ) if scalar @ARGV == 0;
-
-my $pxinject = can_run('pxinject') or die "Can't find pxinject\n";
-my $ppStats = can_run('ppStats') or die "Can't find ppStats\n";
-
-# if workdir is not defined, assign the current path
-# XXX we need to handle relative paths for workdir (not allowed)
-if (! $workdir) {
-    $workdir = File::Spec->rel2abs( "." );
-}
-
-my $num = 0;
-foreach my $file ( @ARGV ) {
-    my $absfile = File::Spec->rel2abs( $file );
-    inject($absfile, $workdir, $dbname);
-    $num ++;
-}
-
-print "$num files injected.\n";
-
-sub inject
-{
-    my $absfile = shift;		# absolute path for this file
-    my $workdir  = shift;		# absolute path for output directory
-    my $dbname = shift;
-
-    my ( $vol, $path, $name ) = File::Spec->splitpath( $absfile );
-    my ( $exp ) = $name =~ /(.*)\.fits/;
-
-    my $relfile = $ipprc->convert_filename_relative( $absfile );
-
-    my $command_type = "$ppStats -concept FPA.OBSTYPE -concept FPA.TELESCOPE -concept FPA.INSTRUMENT $absfile";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command_type, verbose => 1);
-    die "Unable to perform ppStats: $error_code\n" if not $success;
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) 
-        or die "unable to parse metadata config doc";
-
-    my $type;
-    my $telescope;
-    my $instrument;
-    foreach my $row (@$metadata) {
-	if ($row->{name} eq "FPA.TELESCOPE")  { $telescope = $row->{value}; }
-	if ($row->{name} eq "FPA.INSTRUMENT") { $instrument = $row->{value}; } # this entry must return the camera in the ipp/config system which we load
-#	if ($row->{name} eq "FPA.OBSTYPE")    { $type = $row->{value}; }
-    }
-    
-    my $command_exp = "$pxinject -newExp -exp_id $exp -inst $instrument -telescope $telescope -imfiles 1 -workdir $workdir" ; # Command to run
-    if ($dbname) {
-	$command_exp = "$command_exp -dbname $dbname";
-    }
-
-    my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) =
-	run( command => $command_exp, verbose => 1 );
-    die "Unable to inject $exp: $error_code_exp\n" if not $success_exp;
-    
-    my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag
-    my $exp_tag = $line[2];	# The exposure tag
-    
-    my $command_imfile = "$pxinject -newImfile -exp_tag $exp_tag -class fpa -class_id fpa -uri $relfile"; # Command to run
-    if ($dbname) {
-	$command_imfile = "$command_imfile -dbname $dbname";
-    }
-    
-    my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 );
-    die "Unable to inject $exp imfile: $error_code_imfile\n" if not $success_imfile;
-
-    return 1;
-}
-
-
-__END__
Index: trunk/ippScripts/scripts/inject_phase0.pl
===================================================================
--- trunk/ippScripts/scripts/inject_phase0.pl	(revision 11408)
+++ 	(revision )
@@ -1,140 +1,0 @@
-#!/usr/bin/env perl
-
-use warnings;
-use strict;
-
-use IPC::Cmd qw( can_run run );
-use PS::IPP::Metadata::Config;
-use PS::IPP::Metadata::List qw( parse_md_list );
-use Data::Dumper;
-
-use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
-use Pod::Usage qw( pod2usage );
-
-my ($camera,			# Camera used
-    $telescope,			# Telescope used
-    $exp_type,			# Type of exposure
-    $path,			# Path to use
-    $inject_only		# Only inject (i.e., no phase 0)?
-    );
-GetOptions(
-	   'camera|c=s' => \$camera,
-	   'telescope|t=s' => \$telescope,
-	   'exp_type|e=s' => \$exp_type,
-	   'path=s' => \$path,
-	   'inject-only' => \$inject_only,
-) or pod2usage( 2 );
-
-pod2usage(
-	  -msg => "Required options: --camera --telescope --exp_type --path",
-	  -exitval => 3,
-	  ) unless scalar @ARGV == 0 or (defined $camera
-					 and defined $telescope
-					 and defined $exp_type
-					 and defined $path);
-
-my $mdcParser = PS::IPP::Metadata::Config->new;	# Metadata config parser
-
-# Look for programs we need
-my $missing_tools;
-my $pxinject = can_run('pxinject')  or (warn "Can't find pxinject" and $missing_tools = 1);
-my $p0tool = can_run('p0tool') or (warn "Can't find p0tool" and $missing_tools = 1);
-my $phase0_imfile = can_run('phase0_imfile.pl') or (warn "Can't find phase0_imfile.pl" and $missing_tools = 1);
-my $phase0_exp = can_run('phase0_exp.pl') or (warn "Can't find phase0_exp.pl" and $missing_tools = 1);
-die "Can't find required tools.\n" if $missing_tools;
-
-# Inject new data into the database
-if (scalar @ARGV != 0) {
-    my @classes;
-    my $imfiles;
-    if ($camera eq "MEGACAM") {
-	for (my $i = 0; $i < 36; $i++) {
-	    push @classes, sprintf("ccd%02d", $i);
-	}
-    } elsif ($camera eq "MCSHORT") {
-	@classes = ( 'ccd12', 'ccd13', 'ccd14', 'ccd21', 'ccd22', 'ccd23' );
-    } elsif ($camera eq "CTIO_MOSAIC2") {
-	@classes = ();
-    } else {
-	die "Unrecognised camera name: $camera.\n";
-    }
-
-    foreach my $exp ( @ARGV ) {
-	my $command = "$pxinject -newExp -exp_id $exp -inst $camera -telescope $telescope -exp_type $exp_type -imfiles " . (scalar @classes or 1) ; # Command to run
-	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run( command => $command, verbose => 1 );
-	die "Unable to inject $exp: $error_code\n" if not $success;
-	
-	my @line = split(/\s+/, $$stdout_buf[0]); # The output line, containing the exposure tag
-	my $exp_tag = $line[2];	# The exposure tag
-	foreach my $class_id (@classes) {
-	    my $command = "$pxinject -newImfile -exp_tag $exp_tag -class chip -class_id $class_id -uri $path/$exp.$class_id.fits"; # Command to run
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		run( command => $command, verbose => 1 );
-	    die "Unable to inject $exp $class_id: $error_code\n" if not $success;
-	}
-
-	if (scalar @classes == 0) {
-	    my $command = "$pxinject -newImfile -exp_tag $exp_tag -class fpa -class_id fpa -uri $path/$exp.fits"; # Command to run
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		run( command => $command, verbose => 1 );
-	    die "Unable to inject $exp imfile: $error_code\n" if not $success;
-	}
-
-    }
-}
-
-exit 0 if $inject_only;
-
-# Phase 0 imfile processing
-{
-    my $command = "$p0tool -pendingimfile"; # Command to run
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run( command => $command, verbose => 1 );
-    die "Unable to get phase 0 imfile list: $error_code\n" if not $success;
-    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
-	die "Unable to parse output from p0tool.\n";
-
-    foreach my $item (@$list) {
-	my $exp_tag = $item->{exp_tag};
-	my $class = $item->{class};
-	my $class_id = $item->{class_id};
-	my $uri = $item->{uri};
-
-	my $command = "$phase0_imfile --exp_tag $exp_tag --class $class --class_id $class_id --uri $uri";
-	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run( command => $command, verbose => 1 );
-	die "Unable to do phase 0 imfile processing on $exp_tag $class_id: $error_code\n" if not $success;
-    }
-}
-
-# Phase 0 exposure processing
-{
-    my $command = "$p0tool -pendingexp"; # Command to run
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run( command => $command, verbose => 1 );
-    die "Unable to get phase 0 exposure list: $error_code\n" if not $success;
-    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
-	die "Unable to parse output from p0tool.\n";
-  
-    foreach my $item (@$list) {
-	my $exp_tag = $item->{exp_tag};
-
-	my $command = "$phase0_exp --exp_tag $exp_tag";
-	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run( command => $command, verbose => 1 );
-	die "Unable to do phase 0 exposure processing on $exp_tag: $error_code\n" if not $success;
-    }
-}
-
-END {
-    my $status = $?;
-    system("sync") == 0
-        or die "failed to execute sync: $!" ;
-    $? = $status;
-}
-
-
-__END__
-
-
Index: trunk/ippScripts/scripts/ipp_serial_detrend.pl
===================================================================
--- trunk/ippScripts/scripts/ipp_serial_detrend.pl	(revision 11414)
+++ trunk/ippScripts/scripts/ipp_serial_detrend.pl	(revision 11414)
@@ -0,0 +1,274 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use IPC::Cmd qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use Data::Dumper;
+
+my $mdcParser = PS::IPP::Metadata::Config->new;	# Metadata config parser
+
+# Look for programs we need
+my $missing_tools;
+my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
+my $detrend_process_imfile = can_run('detrend_process_imfile.pl')  or (warn "Can't find detrend_process_imfile.pl" and $missing_tools = 1);
+my $detrend_process_exp = can_run('detrend_process_exp.pl') or (warn "Can't find detrend_process_exp.pl" and $missing_tools = 1);
+my $detrend_stack = can_run('detrend_stack.pl') or (warn "Can't find detrend_stack.pl" and $missing_tools = 1);
+my $detrend_norm_calc = can_run('detrend_norm_calc.pl') or (warn "Can't find detrend_norm_calc.pl" and $missing_tools = 1);
+my $detrend_norm_apply = can_run('detrend_norm_apply.pl') or (warn "Can't find detrend_norm_apply.pl" and $missing_tools = 1);
+my $detrend_norm_exp = can_run('detrend_norm_exp.pl') or (warn "Can't find detrend_norm_exp.pl" and $missing_tools = 1);
+my $detrend_resid = can_run('detrend_resid.pl') or (warn "Can't find detrend_resid.pl" and $missing_tools = 1);
+my $detrend_reject_imfile = can_run('detrend_reject_imfile.pl') or (warn "Can't find detrend_reject_imfile.pl" and $missing_tools = 1);
+my $detrend_reject_exp = can_run('detrend_reject_exp.pl') or (warn "Can't find detrend_reject_exp.pl" and $missing_tools = 1);
+die "Can't find required tools.\n" if $missing_tools;
+
+# Process raw imfiles
+{
+    my $command = "$dettool -toprocessedimfile";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get detrend raw list: $error_code\n" if not $success;
+
+    my @whole = split /\n/, join( '', @$stdout_buf );
+    my @single = ();
+
+    while ( scalar @whole > 0 ) {
+	my $value = shift @whole;
+	push @single, $value;
+	if ($value =~ /^\s*END\s*$/) {
+	    push @single, "\n";
+
+	    my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
+		die "Unable to parse output from dettool.\n";
+	    
+	    foreach my $item (@$list) {
+		my $det_id = $item->{det_id};
+		my $det_type = $item->{det_type};
+		my $exp_tag = $item->{exp_tag};
+		my $class = $item->{class};
+		my $class_id = $item->{class_id};
+		my $uri = $item->{uri};
+		my $camera = $item->{camera};
+		
+		my $command = "$detrend_process_imfile --det_id $det_id --exp_tag $exp_tag --class $class --class_id $class_id --det_type $det_type --input_uri $uri --camera $camera";
+		my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		    run( command => $command, verbose => 1 );
+		die "Unable to do raw imfile processing on $exp_tag $class_id: $error_code\n" if not $success;
+	    }
+
+	    @single = ();
+
+	}	
+    }
+}
+
+# Process raw exposures
+{
+    my $command = "$dettool -toprocessedexp";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get detrend raw list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $exp_tag = $item->{exp_tag};
+	my $camera = $item->{camera};
+	my $det_id = $item->{det_id};
+
+	my $command = "$detrend_process_exp --det_id $det_id --exp_tag $exp_tag --camera $camera";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to do raw exposure processing on $det_id $exp_tag: $error_code\n" if not $success;
+    }
+}
+
+# Stack
+{
+    my $command = "$dettool -tostacked";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get stack list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $det_id = $item->{det_id};
+	my $iteration = $item->{iteration};
+	my $class_id = $item->{class_id};
+	my $det_type = $item->{det_type};
+	my $camera = $item->{camera};
+
+	my $command = "$detrend_stack --det_id $det_id --iteration $iteration --class_id $class_id --det_type $det_type --camera $camera";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to stack detrend $det_id $iteration $class_id: $error_code\n" if not $success;
+    }
+}
+
+# Calculate normalisation
+{
+    my $command = "$dettool -tonormalizedstat";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get normalise calculation list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $det_id = $item->{det_id};
+	my $iteration = $item->{iteration};
+	my $det_type = $item->{det_type};
+
+	my $command = "$detrend_norm_calc --det_id $det_id --iteration $iteration --det_type $det_type";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to calculate normalisation for $det_id $iteration: $error_code\n" if not $success;
+    }
+}
+
+# Apply normalisation
+{
+    my $command = "$dettool -tonormalize";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get normalisation list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $det_id = $item->{det_id};
+	my $iteration = $item->{iteration};
+	my $det_type = $item->{det_type};
+	my $class_id = $item->{class_id};
+	my $value = $item->{norm};
+	my $uri = $item->{uri};
+	my $camera = $item->{camera};
+
+	my $command = "$detrend_norm_apply --det_id $det_id --iteration $iteration --class_id $class_id --value $value --input_uri $uri --camera $camera --det_type $det_type";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to apply normalisation for $det_id $iteration $class_id: $error_code\n" if not $success;
+    }
+}
+
+# Examine normalised exposure
+{
+    my $command = "$dettool -tonormalizedexp";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get normalised exposures list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $det_id = $item->{det_id};
+	my $iteration = $item->{iteration};
+	my $det_type = $item->{det_type};
+	my $class_id = $item->{class_id};
+	my $value = $item->{norm};
+	my $uri = $item->{uri};
+	my $camera = $item->{camera};
+
+	my $command = "$detrend_norm_exp --det_id $det_id --iteration $iteration --camera $camera --det_type $det_type";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to examine normalised exposure for $det_id $iteration: $error_code\n" if not $success;
+    }
+}
+
+# Get residuals
+{
+    my $command = "$dettool -toresidimfile";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get residual processing list: $error_code\n" if not $success;
+
+    my @whole = split /\n/, join( '', @$stdout_buf );
+    my @single = ();
+
+    while ( scalar @whole > 0 ) {
+	my $value = shift @whole;
+	push @single, $value;
+	if ($value =~ /^\s*END\s*$/) {
+	    push @single, "\n";
+
+	    my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
+		die "Unable to parse output from dettool.\n";
+	    
+	    foreach my $item (@$list) {
+		my $exp_tag = $item->{exp_tag};
+		my $camera = $item->{camera};
+		my $det_id = $item->{det_id};
+		my $iteration = $item->{iteration};
+		my $class_id = $item->{class_id};
+		my $det_type = $item->{det_type};
+		my $detrend = $item->{det_uri};
+		my $uri = $item->{uri};
+		my $mode = $item->{mode};
+
+		my $command = "$detrend_resid --det_id $det_id --iteration $iteration --exp_tag $exp_tag --class_id $class_id --det_type $det_type --camera $camera --input_uri $uri --mode $mode";
+		$command .= " --detrend $detrend" if defined $detrend;
+
+		my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		    run( command => $command, verbose => 1 );
+		die "Unable to do residual processing on $exp_tag $class_id: $error_code\n" if not $success;
+	    }
+
+	    @single = ();
+
+	}
+    }
+}
+
+# Reject based on imfiles
+{
+    my $command = "$dettool -toresidexp";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get residual imfile list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $exp_tag = $item->{exp_tag};
+	my $camera = $item->{camera};
+	my $det_id = $item->{det_id};
+	my $iteration = $item->{iteration};
+	my $det_type = $item->{det_type};
+
+	my $command = "$detrend_reject_imfile --det_id $det_id --iteration $iteration --exp_tag $exp_tag --det_type $det_type --camera $camera";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to do imfile rejection on $exp_tag $det_id $iteration: $error_code\n" if not $success;
+    }
+}
+
+# Reject based on exposures
+{
+    my $command = "$dettool -residdetrun";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get residual exposure list: $error_code\n" if not $success;
+    my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from dettool.\n";
+
+    foreach my $item (@$list) {
+	my $camera = $item->{camera};
+	my $det_id = $item->{det_id};
+	my $iteration = $item->{iteration};
+	my $det_type = $item->{det_type};
+
+	my $command = "$detrend_reject_exp --det_id $det_id --iteration $iteration --det_type $det_type --camera $camera";
+    	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to do exposure rejection on $det_id $iteration: $error_code\n" if not $success;
+    }
+}
+
+
+__END__
+
+
Index: trunk/ippScripts/scripts/ipp_serial_inject.pl
===================================================================
--- trunk/ippScripts/scripts/ipp_serial_inject.pl	(revision 11414)
+++ trunk/ippScripts/scripts/ipp_serial_inject.pl	(revision 11414)
@@ -0,0 +1,97 @@
+#!/usr/bin/env perl
+
+# this program injects a list of single-file exposures into the db, taking the
+# filename (without .fits) as the exp_tag.
+
+use warnings;
+use strict;
+
+use IPC::Cmd qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use File::Spec;
+use File::Find::Rule;
+use PS::IPP::Config;
+my $ipprc = PS::IPP::Config->new();
+my $mdcParser = PS::IPP::Metadata::Config->new;	# Parser for metadata config files
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+
+# Parse the command-line arguments
+my ($workdir, $dbname);
+GetOptions(
+    'workdir|w=s'   => \$workdir,  # working directory for output files
+    'dbname|d=s'    => \$dbname    # Database name
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Usage: $0 [--workdir path] [--dbname dbname] (files)", -exitval => 2 ) if scalar @ARGV == 0;
+
+my $pxinject = can_run('pxinject') or die "Can't find pxinject\n";
+my $ppStats = can_run('ppStats') or die "Can't find ppStats\n";
+
+# if workdir is not defined, assign the current path
+# XXX we need to handle relative paths for workdir (not allowed)
+if (! $workdir) {
+    $workdir = File::Spec->rel2abs( "." );
+}
+
+my $num = 0;
+foreach my $file ( @ARGV ) {
+    my $absfile = File::Spec->rel2abs( $file );
+    inject($absfile, $workdir, $dbname);
+    $num ++;
+}
+
+print "$num files injected.\n";
+
+sub inject
+{
+    my $absfile = shift;		# absolute path for this file
+    my $workdir  = shift;		# absolute path for output directory
+    my $dbname = shift;
+
+    my ( $vol, $path, $name ) = File::Spec->splitpath( $absfile );
+    my ( $exp ) = $name =~ /(.*)\.fits/;
+
+    my $relfile = $ipprc->convert_filename_relative( $absfile );
+
+    my $command_type = "$ppStats -concept FPA.OBSTYPE -concept FPA.TELESCOPE -concept FPA.INSTRUMENT $absfile";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command_type, verbose => 1);
+    die "Unable to perform ppStats: $error_code\n" if not $success;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) 
+        or die "unable to parse metadata config doc";
+
+    my $type;
+    my $telescope;
+    my $instrument;
+    foreach my $row (@$metadata) {
+	if ($row->{name} eq "FPA.TELESCOPE")  { $telescope = $row->{value}; }
+	if ($row->{name} eq "FPA.INSTRUMENT") { $instrument = $row->{value}; } # this entry must return the camera in the ipp/config system which we load
+#	if ($row->{name} eq "FPA.OBSTYPE")    { $type = $row->{value}; }
+    }
+    
+    my $command_exp = "$pxinject -newExp -exp_id $exp -inst $instrument -telescope $telescope -imfiles 1 -workdir $workdir" ; # Command to run
+    if ($dbname) {
+	$command_exp = "$command_exp -dbname $dbname";
+    }
+
+    my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) =
+	run( command => $command_exp, verbose => 1 );
+    die "Unable to inject $exp: $error_code_exp\n" if not $success_exp;
+    
+    my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag
+    my $exp_tag = $line[2];	# The exposure tag
+    
+    my $command_imfile = "$pxinject -newImfile -exp_tag $exp_tag -class fpa -class_id fpa -uri $relfile"; # Command to run
+    if ($dbname) {
+	$command_imfile = "$command_imfile -dbname $dbname";
+    }
+    
+    my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 );
+    die "Unable to inject $exp imfile: $error_code_imfile\n" if not $success_imfile;
+
+    return 1;
+}
+
+
+__END__
Index: trunk/ippScripts/scripts/ipp_serial_inject_mosaic.pl
===================================================================
--- trunk/ippScripts/scripts/ipp_serial_inject_mosaic.pl	(revision 11414)
+++ trunk/ippScripts/scripts/ipp_serial_inject_mosaic.pl	(revision 11414)
@@ -0,0 +1,91 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use IPC::Cmd qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use Data::Dumper;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+my ($camera,			# Camera used
+    $telescope,			# Telescope used
+    $exp_type,			# Type of exposure
+    $path,			# Path to use
+    );
+GetOptions(
+	   'camera|c=s' => \$camera,
+	   'telescope|t=s' => \$telescope,
+	   'exp_type|e=s' => \$exp_type,
+	   'path=s' => \$path,
+) or pod2usage( 2 );
+
+pod2usage(
+	  -msg => "Required options: --camera --telescope --exp_type --path",
+	  -exitval => 3,
+	  ) unless scalar @ARGV == 0 or (defined $camera
+					 and defined $telescope
+					 and defined $exp_type
+					 and defined $path);
+
+my $mdcParser = PS::IPP::Metadata::Config->new;	# Metadata config parser
+
+# Look for programs we need
+my $missing_tools;
+my $pxinject = can_run('pxinject')  or (warn "Can't find pxinject" and $missing_tools = 1);
+
+# Inject new data into the database
+if (scalar @ARGV != 0) {
+    my @classes;
+    my $imfiles;
+    if ($camera eq "MEGACAM") {
+	for (my $i = 0; $i < 36; $i++) {
+	    push @classes, sprintf("ccd%02d", $i);
+	}
+    } elsif ($camera eq "MCSHORT") {
+	@classes = ( 'ccd12', 'ccd13', 'ccd14', 'ccd21', 'ccd22', 'ccd23' );
+    } elsif ($camera eq "CTIO_MOSAIC2") {
+	@classes = ();
+    } else {
+	die "Unrecognised camera name: $camera.\n";
+    }
+
+    foreach my $exp ( @ARGV ) {
+	my $command = "$pxinject -newExp -exp_id $exp -inst $camera -telescope $telescope -exp_type $exp_type -imfiles " . (scalar @classes or 1) ; # Command to run
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run( command => $command, verbose => 1 );
+	die "Unable to inject $exp: $error_code\n" if not $success;
+	
+	my @line = split(/\s+/, $$stdout_buf[0]); # The output line, containing the exposure tag
+	my $exp_tag = $line[2];	# The exposure tag
+	foreach my $class_id (@classes) {
+	    my $command = "$pxinject -newImfile -exp_tag $exp_tag -class chip -class_id $class_id -uri $path/$exp.$class_id.fits"; # Command to run
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run( command => $command, verbose => 1 );
+	    die "Unable to inject $exp $class_id: $error_code\n" if not $success;
+	}
+
+	if (scalar @classes == 0) {
+	    my $command = "$pxinject -newImfile -exp_tag $exp_tag -class fpa -class_id fpa -uri $path/$exp.fits"; # Command to run
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run( command => $command, verbose => 1 );
+	    die "Unable to inject $exp imfile: $error_code\n" if not $success;
+	}
+
+    }
+}
+
+END {
+    my $status = $?;
+    system("sync") == 0
+        or die "failed to execute sync: $!" ;
+    $? = $status;
+}
+
+
+__END__
+
+
Index: trunk/ippScripts/scripts/ipp_serial_phase0.pl
===================================================================
--- trunk/ippScripts/scripts/ipp_serial_phase0.pl	(revision 11414)
+++ trunk/ippScripts/scripts/ipp_serial_phase0.pl	(revision 11414)
@@ -0,0 +1,86 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use IPC::Cmd qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use Data::Dumper;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+my ($dbname			# Database name to use
+    );
+GetOptions(
+	   'dbname|d=s' => \$dbname,
+) or pod2usage( 2 );
+
+pod2usage(
+	  -msg => "Required options: --dbname",
+	  -exitval => 3,
+	  ) unless defined $dbname;
+
+my $mdcParser = PS::IPP::Metadata::Config->new;	# Metadata config parser
+
+# Look for programs we need
+my $missing_tools;
+my $p0tool = can_run('p0tool') or (warn "Can't find p0tool" and $missing_tools = 1);
+my $phase0_imfile = can_run('phase0_imfile.pl') or (warn "Can't find phase0_imfile.pl" and $missing_tools = 1);
+my $phase0_exp = can_run('phase0_exp.pl') or (warn "Can't find phase0_exp.pl" and $missing_tools = 1);
+die "Can't find required tools.\n" if $missing_tools;
+
+# Phase 0 imfile processing
+my $list;
+{
+    my $command = "$p0tool -pendingimfile -dbname $dbname"; # Command to run
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get phase 0 imfile list: $error_code\n" if not $success;
+    $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from p0tool.\n";
+}
+
+foreach my $item (@$list) {
+    my $exp_tag = $item->{exp_tag};
+    my $class = $item->{class};
+    my $class_id = $item->{class_id};
+    my $uri = $item->{uri};
+    
+    my $command = "$phase0_imfile --exp_tag $exp_tag --class $class --class_id $class_id --uri $uri --dbname $dbname";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to do phase 0 imfile processing on $exp_tag $class_id: $error_code\n" if not $success;
+}
+
+# Phase 0 exposure processing
+{
+    my $command = "$p0tool -pendingexp -dbname $dbname"; # Command to run
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to get phase 0 exposure list: $error_code\n" if not $success;
+    $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or
+	die "Unable to parse output from p0tool.\n";
+}
+
+foreach my $item (@$list) {
+    my $exp_tag = $item->{exp_tag};
+    
+    my $command = "$phase0_exp --exp_tag $exp_tag --dbname $dbname";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run( command => $command, verbose => 1 );
+    die "Unable to do phase 0 exposure processing on $exp_tag: $error_code\n" if not $success;
+}
+
+END {
+    my $status = $?;
+    system("sync") == 0
+        or die "failed to execute sync: $!" ;
+    $? = $status;
+}
+
+
+__END__
+
+