Index: /branches/eam_branches/20091201/ippScripts/Build.PL
===================================================================
--- /branches/eam_branches/20091201/ippScripts/Build.PL	(revision 26720)
+++ /branches/eam_branches/20091201/ippScripts/Build.PL	(revision 26721)
@@ -98,5 +98,7 @@
         scripts/rcserver_checkstatus.pl
         scripts/publish_file.pl
+	scripts/dqstats_bundle.pl
         scripts/whichimage
+        scripts/automate_stacks.pl
     )],
     dist_abstract => 'Scripts for running the Pan-STARRS IPP',
Index: /branches/eam_branches/20091201/ippScripts/MANIFEST
===================================================================
--- /branches/eam_branches/20091201/ippScripts/MANIFEST	(revision 26720)
+++ /branches/eam_branches/20091201/ippScripts/MANIFEST	(revision 26721)
@@ -36,3 +36,4 @@
 scripts/isp_trans.pl
 scripts/ds9_cmf_regions.pl
+scripts/automate_stacks.pl
 t/00_distribution.t
Index: /branches/eam_branches/20091201/ippScripts/scripts/automate_stacks.pl
===================================================================
--- /branches/eam_branches/20091201/ippScripts/scripts/automate_stacks.pl	(revision 26721)
+++ /branches/eam_branches/20091201/ippScripts/scripts/automate_stacks.pl	(revision 26721)
@@ -0,0 +1,600 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+use Carp;
+use IPC::Cmd 0.36 qw( can_run run);
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+
+# Set up
+my $missing_tools = 0;
+my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
+my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1);
+
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# global variables.
+my @target_list = ('M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10');
+my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000');
+my $db;
+my $debug = 0;
+my %metadata_out = ();
+$metadata_out{nsState} = 'UNDEFINEDERROR';
+my $loghead;
+chomp($loghead = `date`);
+print STDERR 'Starting: ' . $loghead . ' ' . $0 . ' ' . (join ' ', @ARGV) . "\n";
+# Grab options
+my ( $date, $camera, $dbname, $logfile, $verbose);
+my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips, $check_stacks, $queue_stacks);
+my ( $help, $this_target_only, $this_filter_only, $isburning, $force_stack_count);
+GetOptions(
+    'help|h'               => \$help,
+    'date=s'               => \$date, # night to make stacks for
+    'camera=s'             => \$camera, # camera 
+    'dbname=s'             => \$dbname, # Database name
+    'logfile=s'            => \$logfile, 
+    'verbose'              => \$verbose,
+    'check_chips'          => \$check_chips,
+    'queue_chips'          => \$queue_chips,
+    'check_stacks'         => \$check_stacks,
+    'queue_stacks'         => \$queue_stacks,
+    'queue_burntool'       => \$queue_burntool,
+    'define_burntool'      => \$define_burntool,
+    'check_registration|R' => \$check_registration,
+    'this_target_only=s'   => \$this_target_only,
+    'this_filter_only=s'   => \$this_filter_only,
+    'isburning'            => \$isburning,
+    'force_stack_count'    => \$force_stack_count,
+    ) or pod2usage ( 2 );
+pod2usage( -msg => 
+"USAGE: automate_stacks.pl <options...> <mode>
+        Options:
+           --help                 This help.
+           --date YYYY-MM-DD      Work on this date (defaults to today GMT).
+           --camera <camera>      Default GPC1.
+           --dbname <db>          Default gpc1.
+           --verbose              
+           --this_target_only     Process only a single target.
+           --this_filter_only     Process only a single filter.
+           --isburning            Signal that we are currently burntooling.
+           --force_stack_count    Force the chip/warp counts.
+        Modes:
+           --check_registration   Confirm the data downloaded correctly.
+           --define_burntool      Determine date ranges for burntool.
+           --queue_burntool       Issue commands to burntool data.
+           --check_chips          Confirm that chips can be built.
+           --queue_chips          Issue chiptool commands to queue chips.
+           --check_stacks         Confirm that stacks can be built.
+           --queue_stacks         Issue stacktool commands to queue stacks.\n",
+	   -exitval => 2, ) if (defined($help));
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage(
+          -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks",
+          -exitval => 3,
+          ) unless
+    defined $check_registration or defined $define_burntool or defined $queue_burntool or
+    defined $queue_chips or defined $queue_stacks or
+    defined $check_chips or defined $check_stacks;
+
+
+unless(defined($date)) {
+    my @trash = gmtime;
+    $trash[5] += 1900;
+    $trash[4] += 1;
+    $date = sprintf("%4d-%02d-%02d",$trash[5],$trash[4],$trash[3]);
+}
+unless(defined($camera)) {
+    $camera = 'GPC1';
+}
+unless(defined($dbname)) {
+    $dbname = 'gpc1';
+}
+
+if (defined($this_target_only)) {
+    foreach my $t (@target_list) {
+	if ($t eq $this_target_only) {
+	    @target_list = ($this_target_only);
+	    last;
+	}
+    }
+    die("$this_target_only is invalid.") if ($#target_list != 0);
+}
+
+if (defined($this_filter_only)) {
+    foreach my $t (@filter_list) {
+	if ($t eq $this_filter_only) {
+	    @filter_list = ($this_filter_only);
+	    last;
+	}
+    }
+    die("$this_filter_only is invalid.") if ($#filter_list != 0);
+}
+
+if (defined($check_registration)) {
+    my ($Nsummit_exp,$Nfaults) = check_summit_copy($date);
+    
+    if ($Nsummit_exp == 0) {
+	print STDERR "No exposures were found on the summit for $date.\n";
+	$metadata_out{nsState} = 'DROP';
+    }
+    elsif ($Nfaults != 0) {
+	print STDERR "There were faults while downloading the exposures for $date.\n";
+	$metadata_out{nsState} = 'NEW';
+    }
+    else {
+	print STDERR "Summit copy and Registration have succeeded for $date.\n";
+	$metadata_out{nsState} = 'REGISTERED';
+    }
+    return_metadata($date);
+    exit(0);
+}
+if (defined($define_burntool)) {
+    $metadata_out{nsState} = 'QUEUEBURNING';
+    &find_burntool_ranges($date);
+    return_metadata($date);
+    exit(0);
+}
+if (defined($queue_burntool)) {
+    $metadata_out{nsState} = 'BURNING';
+    return_metadata($date);
+    exit(0);
+}
+if (defined($check_chips)) {
+    $metadata_out{nsState} = 'QUEUECHIPS';
+    &execute_chips($date,"pretend");
+    if (($metadata_out{nsState} eq 'NEEDSBURNING')&&(defined($isburning))) {
+	$metadata_out{nsState} = 'BURNING';
+    }
+    return_metadata($date);
+    exit(0);
+}
+if (defined($check_stacks)) {
+    $metadata_out{nsState} = 'TOWARP';
+    &execute_stacks($date,"pretend");
+    if ($metadata_out{nsState} eq 'FORCETOWARP') {
+	$metadata_out{nsState} = 'TOWARP';
+    }
+    return_metadata($date);
+    exit(0);
+}
+if (defined($queue_chips)) {
+    $metadata_out{nsState} = 'TOWARP';
+    &execute_chips($date);
+    return_metadata($date);
+    exit(0);
+}
+if (defined($queue_stacks)) {
+    $metadata_out{nsState} = 'STACKING';
+    &execute_stacks($date);
+    return_metadata($date);
+    exit(0);
+}
+
+# Registration 
+
+sub check_summit_copy {
+    my $date = shift;
+    my $db = init_gpc_db();
+
+    # largely stolen from Bill's checkexp program.
+    
+    my $sth = " SELECT exp_name, summitExp.dateobs AS registered, rawExp.dateobs, summitExp.imfiles, ";
+    $sth .= " summitExp.fault AS summit_fault, pzDownloadExp.state AS download_state, ";
+    $sth .= " count(pzDownloadImfile.class_id) AS download_count, newExp.state AS newExp_state, newExp.exp_id, summitExp.exp_type";
+    $sth .= " FROM summitExp LEFT JOIN pzDownloadExp USING(exp_name) LEFT JOIN pzDownloadImfile USING(exp_name) ";
+    $sth .= " LEFT JOIN newExp ON exp_name = tmp_exp_name LEFT JOIN rawExp USING(exp_id, exp_name) ";
+    $sth .= " WHERE date(summitExp.dateobs) >= '${date}T00:00:00' AND date(summitExp.dateobs) <= '${date}T23:59:59' ";
+    $sth .= " GROUP BY exp_name ORDER BY summitExp.dateobs ";
+
+    my $data_ref = $db->selectall_arrayref( $sth );
+    
+    my $Nsummit_exps = 0;
+    my $Nsummit_faults = 0;
+    my $Ndownload_faults = 0;
+    my $Nregister_faults = 0;
+
+#    my_trace($sth,$data_ref,$#{ $data_ref });
+
+    foreach my $row_ref (@{ $data_ref }) {
+	my ($exp_name,$registered,$dateobs,$imfiles,$summit_fault,
+	    $download_state,$download_count,$new_state,$exp_id,$exp_type) = @{ $row_ref };
+	$Nsummit_exps++;
+	if ($summit_fault) {
+	    print STDERR "check_summit_copy: $date $exp_name has summit_fault $summit_fault";
+	    if ($exp_type ne 'OBJECT') {
+		print STDERR " (but I don't care).\n";
+	    }
+	    else {
+		print STDERR "\n";
+		$Nsummit_faults++;
+	    }
+	} 
+	elsif (!$download_state or $download_state eq 'run') {
+	    print STDERR "check_summit_copy: $date $exp_name has download_state $download_state";
+	    if ($exp_type ne 'OBJECT') {
+		print STDERR " (but I don't care).\n";
+	    }
+	    else {
+		print STDERR "\n";
+		$Ndownload_faults++;
+	    }
+	}
+	elsif (!$new_state or $new_state eq 'run' ) {
+	    print STDERR "check_summit_copy: $date $exp_name has new_state $new_state";
+	    if ($exp_type ne 'OBJECT') {
+		print STDERR " (but I don't care).\n";
+	    }
+	    else {
+		print STDERR "\n";
+		$Nregister_faults++;
+	    }
+	}
+    }
+    
+    return($Nsummit_exps,$Nsummit_faults + $Ndownload_faults + $Nregister_faults);
+}	 
+
+# Burntool
+
+sub get_goodBTvalue {
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+
+    my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL.STATE.GOOD | uniq";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run ( command => $config_cmd, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform ppConfigDump: $error_code", 0, 0, 0, $PS_EXIT_SYS_ERROR);
+    }
+    my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata config doc", 0, 0, 0, $PS_EXIT_SYS_ERROR);
+    my $burntoolStateGood = 999;
+    foreach my $cfg (@$recipeData) {
+	if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') {
+	    $burntoolStateGood = $cfg->{value};
+	}
+    }
+    if ($burntoolStateGood == 999) {
+	&my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, 0, 0, $PS_EXIT_SYS_ERROR);
+    }
+    return($burntoolStateGood);
+}
+
+sub verify_burntool {
+    my $exp_id = shift;
+    my $burntoolStateGood = shift;
+
+    my $db = init_gpc_db();
+    
+    my $sth = "SELECT exp_id,exp_name,obs_mode,dateobs,class_id,burntool_state,comment FROM rawImfile WHERE exp_id = $exp_id";
+    my $data_ref = $db->selectall_arrayref( $sth );
+
+    my $Nimfiles = 0;
+    my $Nburntooled = 0;
+
+    foreach my $row_ref (@{ $data_ref }) {
+        my ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$burntool_state,$comment) = @{ $row_ref };
+	$Nimfiles++;
+	if (abs($burntool_state) == $burntoolStateGood) {
+	    $Nburntooled++;
+	}
+    }
+    return($Nimfiles,$Nburntooled);
+}    
+
+sub find_burntool_ranges {
+    my $date = shift;
+    # Much cleaner than reimplementing that mess here
+    my $command = "$mkBTpcontrol -d $date -b";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run ( command => $command, verbose => $verbose );
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform $command: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+    }
+
+    if ($#{ $stderr_buf } > -1) {
+	$metadata_out{nsState} = 'ERROR';	
+	return(1);
+    }
+
+    my $N = 0;
+    my @burntool_entries = split /\n/, (join '', @$stdout_buf);
+    foreach my $row (@burntool_entries) {
+	my ($trash,$start,$end);
+	($trash,$trash,$start,$end) = (split /\s+/,$row);
+	$metadata_out{"btBegin${N}"} = $start;
+	$metadata_out{"btEnd${N}"} = $end;
+	$metadata_out{"btCounter${N}"} = 0;
+	$N++;
+	print STDERR "define_burntool: $row\n";
+    }
+    $metadata_out{btN} = $N - 1;
+    $metadata_out{btNCounter} = 0;
+
+    return(0);
+}
+    
+
+# Chips
+
+sub construct_chiptool_cmd {
+    my $date = shift;
+    my $target = shift;
+    
+    my $select =  "-dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 ";
+    $date =~ s/-//g;
+
+    my $cmd = "$chiptool";
+    $cmd .= ' -simple -dbname gpc1 -definebyquery -set_end_stage warp ';
+    $cmd .= " -set_label ${target}.nightlystack ";
+    $cmd .= " -set_workdir neb://\@HOST\@.0/gpc1/${target}.nightlystack -set_dist_group $target ";
+    $cmd .= " -set_tess_id $target -comment ${target}% -set_data_group ${target}.${date} ";
+    $cmd .= " $select ";
+    if ($debug == 1) {
+	$cmd .= " -pretend ";
+    }
+    print STDERR "$cmd\n";
+    return($cmd);
+}
+
+sub verify_uniqueness_chip {
+    my $exp_id = shift;
+    my $date = shift;
+    my $target = shift;
+    
+    my $db = init_gpc_db();
+    $date =~ s/-//g;
+    my $data_group = "${target}.${date}";
+
+    my $sth = "SELECT exp_id from chipRun where data_group = '$data_group' AND exp_id = $exp_id";
+    my $data_ref = $db->selectall_arrayref( $sth );
+
+    return($#{ $data_ref } + 1);
+}
+
+sub pre_chip_queue {
+    my $date = shift;
+    my $target = shift;
+    
+    my $command = construct_chiptool_cmd($date,$target) . ' -pretend ';
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run ( command => $command, verbose => $verbose );
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+    }
+
+    my $burntoolStateGood = get_goodBTvalue();
+    my $Nimfiles = 0;
+    my $Nburntooled = 0;
+    my $Nalready = 0;
+    my @input_exposures = split /\n/, (join '', @$stdout_buf);
+    foreach my $entry (@input_exposures) {
+	my ($exp_id, @trash) = split /\s+/, $entry;
+	@trash = verify_burntool($exp_id,$burntoolStateGood);
+	$Nimfiles += $trash[0];
+	$Nburntooled += $trash[1];
+
+	@trash = verify_uniqueness_chip($exp_id,$date,$target);
+	$Nalready += $trash[0];
+    }
+    return($#input_exposures + 1,$Nimfiles,$Nburntooled,$Nalready);
+}
+
+sub chip_queue {
+    my $date = shift;
+    my $target = shift;
+
+    my $command = construct_chiptool_cmd($date,$target);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run ( command => $command, verbose => $verbose );
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+    }
+    $metadata_out{nsState} = 'TOWARP';
+    return(0);
+}
+
+sub execute_chips {
+    my $date = shift;
+    my $pretend = shift;
+    foreach my $target (@target_list) {
+	my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = pre_chip_queue($date,$target);
+	if ($Nexposures == 0) {
+	    print STDERR "execute_chips: Target $target on $date had no exposures.\n";
+	    next;
+	}
+	if ($Nalready != 0) {
+	    print STDERR "execute_chips: Not queueing $target on $date due to already existing exposures.\n";
+	    next;
+	}
+	if ($Nimfiles != $Nburntooled) {
+	    print STDERR "execute_chips: Target $target on $date is not fully burntooled.\n";
+	    $metadata_out{nsState} = 'NEEDSBURNING';
+	    next;
+	}
+	unless(defined($pretend)) {
+	    chip_queue($date,$target);
+	}
+    }
+}
+
+# Stacking
+
+sub construct_stacktool_cmd {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+
+    my $select =  "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 ";
+    $date =~ s/-//g;
+
+    my $cmd = "$stacktool";
+#    $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 ';  # Probably silly, but I want to be safe and not overwrite
+    $cmd .= ' -simple -dbname gpc1 -definebyquery ';  
+    $cmd .= " -set_label ${target}.nightlystack -select_label ${target}.nightlystack ";
+    $cmd .= " -set_workdir neb://\@HOST\@.0/gpc1/${target}.nightlystack -set_dist_group $target ";
+    $cmd .= " -select_filter $filter -set_data_group ${target}.${date} ";
+    $cmd .= " $select ";
+    if ($debug == 1) {
+	$cmd .= ' -pretend ';
+    }
+    print STDERR "$cmd\n";
+    return($cmd);
+}    
+
+sub verify_uniqueness_stack {
+    my $skycell = shift;
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+
+    my $db = init_gpc_db();
+    $date =~ s/-//g;
+    my $data_group = "${target}.${date}";
+
+    my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter'";
+    my $data_ref = $db->selectall_arrayref( $sth );
+    
+    return($#{ $data_ref } + 1);
+}
+
+sub pre_stack_queue {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+
+    # check warp stage == chip stage
+    my $db = init_gpc_db();
+
+    my $trunc_date = $date; $trunc_date =~ s/-//g;
+
+    my $where = " label = '${target}.nightlystack' AND data_group = '${target}.${trunc_date}' ";
+    my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where ";
+    my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where ";
+
+    my $chip_ref = $db->selectall_arrayref( $chip_sth );
+    my $warp_ref = $db->selectall_arrayref( $warp_sth );
+
+    # check that we will be able to queue them up
+    my $command = construct_stacktool_cmd($date,$target,$filter) . ' -pretend ';
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+	run ( command => $command, verbose => $verbose );
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform stacktool: $error_code", $#{ $chip_ref },$#{ $warp_ref },$date, $PS_EXIT_SYS_ERROR);
+    }
+    
+    my $Nalready = 0;
+
+    my @input_exposures = split /\n/, (join '', @$stdout_buf);    
+    foreach my $entry (@input_exposures) {
+	my ($skycell, @trash) = split /\s+/, $entry;
+	@trash = verify_uniqueness_stack($skycell,$date,$target,$filter);
+	$Nalready += $trash[0];
+    }
+
+    return($#input_exposures  + 1, $#{ $chip_ref } + 1, $#{ $warp_ref } + 1, $Nalready);
+}
+
+sub stack_queue {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+
+    my $command = construct_stacktool_cmd($date,$target,$filter);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+	run ( command => $command, verbose => $verbose );
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+    }
+    return(0);
+}
+
+sub execute_stacks {
+    my $date = shift;
+    my $pretend = shift;
+    foreach my $target (@target_list) {
+	foreach my $filter (@filter_list) {
+	    my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
+	    if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
+		print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
+		$metadata_out{nsState} = 'FORCETOWARP';
+		next;
+	    }
+	    if ($Nexposures == 0) {
+		print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n";
+		next;
+	    }
+	    if ($Nalready != 0) {
+		print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
+		next;
+	    }
+	    unless ($metadata_out{nsState} eq 'FORCETOWARP') {
+		$metadata_out{nsState} = 'QUEUESTACKS';
+	    }
+	    unless(defined($pretend)) {
+		print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n";
+		stack_queue($date,$target,$filter);
+	    }
+	}
+    }
+    
+}
+
+# Utilities
+
+sub init_gpc_db {
+    use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
+    my $dbserver = 'ippdb01';
+    my $dbuser = 'ippuser';
+    my $dbpass = 'ippuser';
+    $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" .
+		       "mysql_socket=" . DB_SOCKET(),
+		       ${dbuser},${dbpass}, 
+		       { RaiseError => 1, AutoCommit => 1}
+	) or die "Unable to connect to database $DBI::errstr\n";
+    return($db);
+}
+
+sub return_metadata {
+    my $date = shift;
+    print STDOUT "autoStack MULTI\n\n";
+    print STDOUT "autoStack METADATA\n";
+    print STDOUT "   date                  STR          $date\n";
+    foreach my $k (keys %metadata_out) {
+	print STDOUT "   $k                STR          $metadata_out{$k}\n";
+    }
+    print STDOUT "END\n";
+}
+
+sub my_die {
+    my $msg = shift; # Warning message on die
+    my $stage = shift; # stage name
+    my $stage_id = shift; #  identifier
+    my $exit_code = shift; # Exit code
+    # outputImage and path_base are globals
+
+    carp($msg);
+    exit $exit_code;
+}
+
+sub my_trace {
+    if ($debug == 1) {
+	foreach my $thing (@_) {
+	    carp($thing);
+	}
+    }
+}
Index: /branches/eam_branches/20091201/ippScripts/scripts/dqstats_bundle.pl
===================================================================
--- /branches/eam_branches/20091201/ippScripts/scripts/dqstats_bundle.pl	(revision 26721)
+++ /branches/eam_branches/20091201/ippScripts/scripts/dqstats_bundle.pl	(revision 26721)
@@ -0,0 +1,162 @@
+#! /usr/bin/env perl
+
+use warnings;
+use strict;
+use Carp;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use DateTime;
+my $mjd_start = DateTime->now->mjd;   # MJD of starting script
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+use File::Temp qw( tempfile );
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Look for programs we need
+my $missing_tools;
+my $dqstatstool = can_run('dqstatstool') or (warn "Can't find dqstatstool" and $missing_tools = 1);
+my $dsreg       = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+my ( $exp_tag, $dqstats_id, $camera, $uri, $dbname, $verbose, $no_update, $no_op, $redirect, $save_temps);
+GetOptions(
+	   'exp_tag=s'       => \$exp_tag, # exposure identifier
+	   'dqstats_id=s'    => \$dqstats_id, # dqstatsrun ID
+	   'camera|c=s'      => \$camera,  # camera
+	   'dbname|d=s'      => \$dbname,  # database name
+	   'uri=s'           => \$uri, # output file destination
+	   'verbose'         => \$verbose, # print to stdout
+	   'no-update'       => \$no_update, # Update the database
+	   'no-op'           => \$no_op,   # don't do the operations
+	   'redirect-output' => \$redirect, 
+	   'save-temps'      => \$save_temps, # save temporary files
+	   ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options --exp_tag --dqstats_id --camera",
+	   -exitval => 3,
+	   ) unless
+    defined $exp_tag and
+    defined $dqstats_id and
+    defined $camera;
+
+my $ipprc = PS::IPP::Config->new( $camera ) 
+    or my_die( "Unable to set up", $dqstats_id, $PS_EXIT_CONFIG_ERROR ); # IPP config
+#my $logDest = $ipprc->filename("LOG.EXP", $uri) 
+#    or my_die("Missing entry from camera config", $dqstats_id, $PS_EXIT_CONFIG_ERROR);
+
+if ($redirect) {
+#    $ipprc->redirect_output($logDest) 
+#	or my_die( "Unable to redirect output", $dqstats_id, $PS_EXIT_SYS_ERROR );
+    print "\n\n";
+    print "Starting script $0 on $host\n\n";
+    print "COMMAND IS: @ARGV\n\n";
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+# Output products
+unless (defined($uri)) {
+    $uri = "/data/${host}.0/tmp/dqstats.${dqstats_id}.fits";
+}
+#$ipprc->outroot_prepare($uri); # hm....need to think more here.
+
+my $bundle_command = "$dqstatstool -dbname $dbname  -createbundle ";
+$bundle_command .= "-dqstats_id $dqstats_id ";
+$bundle_command .= "-uri $uri ";
+
+my $filesetID = "dqstats.${dqstats_id}";
+
+unless ($no_op) {
+    # This bit needs to make the bundle.
+    
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $bundle_command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to create the bundle.: $error_code", $dqstats_id, $error_code);
+    }
+    
+    # Parse stdout_buf to get output file name
+    ## Begin partial
+    my ($tempFile, $tempName) = tempfile( "/tmp/dqstats.XXXX",
+					  UNLINK => !$save_temps, SUFFIX => 'dsin' );
+    print $tempFile $uri . '|||notset' . "\n";
+
+    my $register_cmd = "$dsreg --product 'DQstats' --add $filesetID --type table --list $tempName --abspath --copy";
+
+    # It can't be that simple, can it? This should probably add a fault on failure.
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $register_cmd, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to register the bundle.: $error_code", $dqstats_id, $error_code);
+    }
+    
+    # We no longer need the file generated by dqstatstool.
+    unlink($uri);
+}
+
+my $update_command = "$dqstatstool -dbname $dbname  -dqstats_id $dqstats_id";
+$update_command .= " -updaterun -set_state 'full' ";
+
+unless ($no_update) {
+    # This bit needs to set the database state to full
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $update_command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	warn("Unable to add result to database: $error_code\n");
+	exit($error_code);
+    }
+} else {
+    print "skipping command: $update_command\n";
+}
+
+
+sub my_die
+{
+    my $msg = shift; # Warning message on die
+    my $dqstats_id = shift; # Camtool identifier
+    my $exit_code = shift; # Exit code to add
+
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+    carp($msg);
+    if (defined $dqstats_id and not $no_update) {
+	# There is no fault handling, which may be a mistake. This is where a fault coding would be.
+#         my $command = "$addtool -add_id $add_id";
+#         $command .= " -addprocessedexp";
+#         $command .= " -fault $exit_code";
+#         $command .= " -path_base $outroot" if defined $outroot;
+#         $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400));
+#         $command .= " -dbname $dbname" if defined $dbname;
+#         system ($command);
+    }
+    exit $exit_code;
+}
+
+END {
+    my $status = $?;
+    system("sync") == 0
+        or die "failed to execute sync: $!" ;
+    $? = $status;
+}
+
+__END__
Index: /branches/eam_branches/20091201/ippScripts/scripts/ipp_cleanup.pl
===================================================================
--- /branches/eam_branches/20091201/ippScripts/scripts/ipp_cleanup.pl	(revision 26720)
+++ /branches/eam_branches/20091201/ippScripts/scripts/ipp_cleanup.pl	(revision 26721)
@@ -190,5 +190,6 @@
 	    # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
 	    my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-	    $command = "$magicdstool -dbname gpc1  -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+	    $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+            $command .= " -dbname $dbname" if defined $dbname;
 	    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 		run(command => $command, verbose => $verbose);
@@ -309,5 +310,6 @@
 	# Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
 	my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-	$command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+	$command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+        $command .= " -dbname $dbname" if defined $dbname;
 	( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 	    run(command => $command, verbose => $verbose);
@@ -444,5 +446,6 @@
 	    # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
 	    my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-	    $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+	    $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+            $command .= " -dbname $dbname" if defined $dbname;
 	    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 		run(command => $command, verbose => $verbose);
@@ -580,5 +583,6 @@
 	    # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
 	    my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-	    $command = "$magicdstool -dbname gpc1  -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+	    $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+            $command .= " -dbname $dbname" if defined $dbname;
 	    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 		run(command => $command, verbose => $verbose);
@@ -732,5 +736,6 @@
 	    # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
 	    my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-	    $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+	    $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
+            $command .= " -dbname $dbname" if defined $dbname;
 	    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 		run(command => $command, verbose => $verbose);
Index: /branches/eam_branches/20091201/ippScripts/scripts/magic_destreak.pl
===================================================================
--- /branches/eam_branches/20091201/ippScripts/scripts/magic_destreak.pl	(revision 26720)
+++ /branches/eam_branches/20091201/ippScripts/scripts/magic_destreak.pl	(revision 26721)
@@ -15,5 +15,5 @@
 
 use IPC::Cmd 0.36 qw( can_run run );
-use File::Temp qw( tempfile );
+use File::Temp qw( tempfile tempdir );
 use File::Basename qw( basename dirname );
 use PS::IPP::Metadata::Config;
@@ -32,4 +32,5 @@
 my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
 my $censorObjects = can_run('censorObjects') or (warn "Can't find censorObjects" and $missing_tools = 1);
+my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -38,5 +39,5 @@
 
 # Parse the command-line arguments
-my ($magic_ds_id, $camera, $streaks, $inv_streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
+my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
 my ($outroot, $recoveryroot);
 my ($replace, $release);
@@ -48,4 +49,5 @@
            'streaks=s'      => \$streaks,    # file containing the list of streaks
            'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
+           'exp_id=s'       => \$exp_id,     # exp_id, chip_id, warp_id, or diff_id
            'stage=s'        => \$stage,      # raw, chip, warp, or diff
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
@@ -74,4 +76,5 @@
     defined $stage and
     defined $stage_id and
+    defined $exp_id and
     defined $component and
     defined $uri and
@@ -145,4 +148,6 @@
     }
 }
+
+my $statsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.stats";
 
 my $backup_path_base;
@@ -166,5 +171,5 @@
         if (! -e $recoveryroot ) {
             my $code = system "mkdir -p $recoveryroot";
-            &my_die("cannot create output directory $outroot", $magic_ds_id, $component,
+            &my_die("cannot create output directory $recoveryroot", $magic_ds_id, $component,
                     $code >> 8) if $code;
         }
@@ -173,4 +178,6 @@
     $recovery_path_base = "$recoveryroot/$basename/";
 }
+
+my $temp_dir;
 
 if ($stage ne "camera") {
@@ -195,11 +202,21 @@
                     &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
 
+            $temp_dir = tempdir( CLEANUP => !$save_temps);
             ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
 
             foreach my $skycell (@$skycells) {
-                my $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+                my $skycell_uri;
+                if ($skycell->{data_state} eq "full") {
+                    $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+                } else {
+                    # diff run must have been cleaned up, need to create this skycell file on the fly
+                    my $skycell_id = $skycell->{skycell_id};
+                    $skycell_uri = "$temp_dir/$skycell_id";
+                    $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
+                        &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+                }
                 print $sfh "$skycell_uri\n";
             }
-            close $sfh;
+            close $sfh
         }
     }
@@ -260,4 +277,5 @@
         my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
 
+        $command .= " -stats $statsFile";
         $command .= " -class_id $class_id" if defined $class_id;
         $command .= " -recovery $recoveryroot" if defined $recoveryroot;
@@ -288,5 +306,9 @@
         $sources = $ipprc->filename("PPSUB.INVERSE.SOURCES", $path_base);
 
+        # Note: we create a stats file for the inverse procesing but we don't look at the results
+        my $invStatsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.inv.stats";
+
         my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
+        $command .= " -stats $invStatsFile";
 
         $command .= " -recovery $recoveryroot" if defined $recoveryroot;
@@ -371,5 +393,25 @@
 }
 
-# if recovery and/or backup files were expected make sure they exist
+my $statsFlags;
+if ($stage ne "camera") {
+    file_check($statsFile);
+    {
+        my $resolvedStatsFile = $ipprc->file_resolve($statsFile);
+        my $command = "$ppStatsFromMetadata $resolvedStatsFile - STREAKSREMOVE";
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform ppStatsFromMetadata: $error_code", $magic_ds_id, $component, $error_code);
+        }
+        foreach my $line (@$stdout_buf) {
+            $statsFlags .= " $line";
+        }
+        chomp $statsFlags;
+    }
+}
+
+# XXX: if recovery and/or backup files were expected make sure they exist
 
 # Input result into database
@@ -381,4 +423,5 @@
     $command   .= " -backup_path_base $backup_path_base" if $backup_path_base;
     $command   .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
+    $command   .= " $statsFlags" if $statsFlags;
     $command   .= " -dbname $dbname" if defined $dbname;
 
Index: /branches/eam_branches/20091201/ippScripts/scripts/magic_destreak_cleanup.pl
===================================================================
--- /branches/eam_branches/20091201/ippScripts/scripts/magic_destreak_cleanup.pl	(revision 26720)
+++ /branches/eam_branches/20091201/ippScripts/scripts/magic_destreak_cleanup.pl	(revision 26721)
@@ -183,5 +183,5 @@
         }
 
-        delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom);
+        delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
 
         if ($stage eq "diff" and $inverse) {
@@ -199,5 +199,6 @@
                 $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base);
             }
-            delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources, undef);
+            # undef is not necessary, it's just to keep the lists in the same order.
+            delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources);
         }
 }
@@ -211,5 +212,5 @@
 if (!$no_update) {
     my $result = $dbh->do("UPDATE magicDSRun SET state = 'cleaned' WHERE magic_ds_id = ?", undef, $magic_ds_id);
-    my_die("attempt to update magicDSRun failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0";
+    my_die("attempt to update magicDSRun.state failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0";
 } else {
     print STDERR "skipping update of magicDSRun\n";
@@ -246,5 +247,5 @@
     $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
 
-    my $command = "$magicdstool -updaterun -fault $exit_code";
+    my $command = "$magicdstool -updaterun -set_state failed_cleanup";
     $command   .= " -magic_ds_id $magic_ds_id";
     $command   .= " -dbname $dbname" if defined $dbname;
Index: /branches/eam_branches/20091201/ippScripts/scripts/magic_destreak_revert.pl
===================================================================
--- /branches/eam_branches/20091201/ippScripts/scripts/magic_destreak_revert.pl	(revision 26720)
+++ /branches/eam_branches/20091201/ippScripts/scripts/magic_destreak_revert.pl	(revision 26721)
@@ -294,4 +294,8 @@
         run(command => $command, verbose => $verbose);
 
+    if (!defined $error_code) {
+        print STDERR "run( $command ) returned undef error_code!!!!!\n";
+        return $PS_EXIT_UNKNOWN_ERROR;
+    }
     return $error_code >> 8;
 }
@@ -392,5 +396,5 @@
     my $command = "$magicdstool -updaterun";
     $command   .= " -magic_ds_id $magic_ds_id";
-    $command   .= " -fault $exit_code";
+    $command   .= " -set_state failed_revert";
     $command   .= " -dbname $dbname" if defined $dbname;