Index: trunk/ippScripts/scripts/diff_skycell.pl
===================================================================
--- trunk/ippScripts/scripts/diff_skycell.pl	(revision 31422)
+++ trunk/ippScripts/scripts/diff_skycell.pl	(revision 31435)
@@ -31,4 +31,5 @@
 my $ppSub = can_run('ppSub') or (warn "Can't find ppSub" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $nebInsert = can_run('neb-insert') or (warn "Can't find neb-insert" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -265,4 +266,7 @@
 my $do_photom = 1;
 my $dump_config = 1; 
+if ($reduction eq 'NOCONVDIFF') {
+    $do_photom = 0;
+}
 if ($run_state eq 'new') {
     $configuration = prepare_output("PPSUB.CONFIG", $outroot, 1);
@@ -380,4 +384,17 @@
                 }
             }
+	    if ($reduction eq 'NOCONVDIFF') {
+		my $refConv = prepare_output("PPSUB.REF.CONV", $outroot, 0);
+		my $templateFile = $ipprc->file_resolve($template,0);
+		$command = "$nebInsert $refConv $templateFile";
+		( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		    run(command => $command, verbose => $verbose);
+		unless ($success) {
+		    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		    &my_die("Unable to perform neb-insert: $error_code", $diff_id, $skycell_id, $error_code);
+		}
+	    }
+
+
         } elsif ($run_state eq 'update') {
             &my_die("Update resulted in poor quality image: $quality", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR);
Index: trunk/ippScripts/scripts/lap_science.pl
===================================================================
--- trunk/ippScripts/scripts/lap_science.pl	(revision 31435)
+++ trunk/ippScripts/scripts/lap_science.pl	(revision 31435)
@@ -0,0 +1,907 @@
+#!/usr/bin/env perl
+
+
+use warnings;
+use strict;
+use Carp;
+use IPC::Cmd 0.36 qw( can_run run);
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use DateTime;
+
+#
+# Set up
+################################################################################
+
+my $missing_tools = 0;
+my $regtool  = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
+my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
+my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
+my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
+my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1);
+my $laptool  = can_run('laptool') or (warn "Can't find laptool" and $missing_tools = 1);
+
+my ( $help, $verbose, $debug, $do_nothing);
+my ( $camera, $dbname);
+my ( $lap_id );
+my ( $chip_mode, $monitor_mode, $cleanup_mode);
+
+GetOptions(
+    'help|h'       => \$help,
+    'verbose'      => \$verbose,
+    'debug'        => \$debug,
+    'do_nothing'   => \$do_nothing,
+
+    'camera=s'     => \$camera,
+    'dbname=s'     => \$dbname,
+    
+    'lap_id=s'     => \$lap_id,
+
+    'chip_mode'    => \$chip_mode,
+    'monitor_mode' => \$monitor_mode,
+    'cleanup_mode' => \$cleanup_mode,
+    ) or pod2usage ( 2 );
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage(
+    -msg => "Choose a mode: --chip_mode --monitor_mode --cleanup_mode",
+    -exitval => 3,
+    ) unless
+    defined $chip_mode or defined $monitor_mode or defined $cleanup_mode;
+pod2usage(
+    -msg => "--lap_id is required",
+    -exitval => 3,
+    ) unless
+    defined $lap_id;
+
+unless (defined $dbname) {
+    $dbname = 'gpc1';
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new;
+
+# Fetch all the information about this run
+my %lapRunInfo = get_lapRun_info($lap_id);
+
+# Run the appropriate mode
+
+if (defined($chip_mode)) {
+    unless ($lapRunInfo{state} eq 'new') {
+	&my_die("Cannot run chip_mode if lapRun.state != new!",$lap_id);
+    }
+    my $status = chip_mode($lap_id);
+    exit $status;
+}
+if (defined($monitor_mode)) {
+    unless ($lapRunInfo{state} eq 'run') {
+	&my_die("Cannot run monitor_mode if lapRun != run!", $lap_id);
+    }
+    my $status = monitor_mode($lap_id);
+    exit $status;
+}
+if (defined($cleanup_mode)) {
+    unless (($lapRunInfo{state} eq 'full')||
+	    ($lapRunInfo{state} eq 'drop')) {
+	&my_die("Cannot run cleanup_mode if lapRun != done!", $lap_id);
+    }
+    my $status = cleanup_mode($lap_id);
+    exit $status;
+}
+
+#
+# Chip
+################################################################################
+
+sub chip_mode {
+    my $lap_id = shift;
+    my $status = queue_chips($lap_id);
+
+    if ($status) {
+	my $command = "$laptool -updaterun -lap_id $lap_id";
+	$command .= " -dbname $dbname " if defined $dbname;
+	$command .= " -set_state run ";
+	my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $command, verbose => $verbose);
+	unless ($success) {
+	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	    &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id);
+	}
+    }
+    return($status);
+}
+
+# retrieve the chip_id for the exposure inserted
+sub get_chip_id_from_metadata {
+    my $exp_id = shift;
+    my $label = shift;
+    my $data_group = shift;
+    # This is a puzzler... chiptool doesn't actually return a useful metadata.  We'll just scrape it from the database for now.
+
+    my $command = "$chiptool -listrun -exp_id $exp_id -label $label -data_group $data_group";
+    $command .= " -dbname $dbname " if defined $dbname;
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform chiptool -listrun: $error_code", $exp_id, $data_group);
+    }
+    my $chips = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from chiptool -listrun", $exp_id, $data_group);
+    # There should be only one.
+    my $chip = ${ $chips }[0];
+    my $chip_id = 0;
+    if ($chip) {
+	$chip_id = $chip->{chip_id};
+    }
+    
+    return($chip_id);
+}
+
+sub get_lapRun_info {
+    my $lap_id = shift;
+    my $command = "$laptool -pendingrun -lap_id $lap_id";
+    $command .= " -dbname $dbname " if defined $dbname;
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform laptool -pendingrun: $error_code", $lap_id);
+    }
+    my $Runs = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from laptool -pendingrun", $lap_id);
+    # There should be only one.
+    my $Run = ${ $Runs }[0];
+    my %info = %{ $Run };
+    return(%info);
+}
+
+
+# Queue a chipRun for this exposure with the appropriate parameters.
+sub remake_this_exposure {
+    my $exposure = shift;
+
+    my @utctime = gmtime();
+    $utctime[5] += 1900;
+    $utctime[4] += 1;
+
+    my $label = $exposure->{label};
+
+    my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]);
+    my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
+    my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
+    my $data_group = "${label}.${date}";
+
+
+    my $chip_cmd = "$chiptool";
+    $chip_cmd .= " -pretend " if defined $debug;
+    $chip_cmd .= " -dbname $dbname " if defined $dbname;
+    $chip_cmd .= " -definebyquery -exp_id $exposure->{exp_id} -set_end_stage warp -set_tess_id $exposure->{tess_id} ";
+    $chip_cmd .= " -set_label $exposure->{label} -set_data_group $data_group ";
+    $chip_cmd .= " -set_workdir $workdir -set_dist_group $exposure->{dist_group} ";
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $chip_cmd, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform chiptool -definebyquery: $error_code", $exposure->{lap_id}, $exposure->{proj_cell});
+    }
+    
+    $exposure->{chip_id} = get_chip_id_from_metadata($exposure->{exp_id},$exposure->{label},$data_group); 
+    $exposure->{data_state} = 'new';
+    update_this_exposure($exposure);
+    return($exposure);
+}
+
+# This is the "user level" subroutine.
+sub queue_chips {
+    my $lap_id = shift;
+    
+    my $command = "$laptool -pendingexp -lap_id $lap_id";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform laptool -pendingexp: $error_code", $lap_id);
+    }
+    
+    if (@$stdout_buf == 0) {
+	# Nothing to do.
+	return(0);
+    }
+    
+    my $exposures = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from laptool -pendingexp", $lap_id);
+
+    my $counter = 0;
+    my %matching = ();
+    my %indexing = ();
+    # Determine which exposures need a chipRun queued.
+    foreach my $exposure (@$exposures) {  
+	# $lap_id = $exposure->{lap_id};  # This should be already known.
+	my $tess_id = $exposure->{tess_id};
+	my $exp_id = $exposure->{exp_id};
+	my $chip_id = $exposure->{chip_id};
+	my $pair_id = $exposure->{pair_id};
+	my $private = $exposure->{private};
+	my $pairwise = $exposure->{pairwise};
+	my $active = $exposure->{active};
+	my $data_state = $exposure->{data_state};
+	my $dateobs = $exposure->{dateobs};
+	my $object = $exposure->{object};
+	my $comment = $exposure->{comment};
+
+	if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id
+	    if (($pairwise) && !($pair_id)) {
+		# This is an error.
+		&my_die("Exposure $exp_id for $lap_id is declared pairwise without a defined pair", $lap_id);
+	    }
+	    $matching{$object}{$comment} = $exp_id;
+	    $indexing{$exp_id} = $counter;
+	    $counter++; 
+	    next;
+	}
+	else { # We do not already have a chip_id.  
+	    # Make a chipRun, and update the exposure.
+	    $exposure = remake_this_exposure($exposure); 
+	    
+	    # Save our information for diff pairing.
+	    $matching{$object}{$comment} = $exp_id;
+	    $indexing{$exp_id} = $counter;
+	    $counter++;
+	}
+    }
+
+    # Determine which exposures have a pair for diffing.
+    foreach my $object (keys %matching) { 
+	my @exp_ids_to_diff = ();
+	foreach my $comment (keys %{ $matching{$object} }) {
+	    push @exp_ids_to_diff, $matching{$object}{$comment};
+	}
+	@exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff;
+	
+	if (( $#exp_ids_to_diff + 1) % 2 != 0) { # We have an odd number of exposures, even after comment filtering
+	    pop(@exp_ids_to_diff); # dump the last entry.
+	}
+
+	while ($#exp_ids_to_diff > -1) {  # Save the pair information to the opposite exposure.
+	    my $exp_id_A = shift(@exp_ids_to_diff);
+	    my $exp_id_B = shift(@exp_ids_to_diff);
+
+	    my $exp_A = ${ $exposures }[$indexing{$exp_id_A}];
+	    my $exp_B = ${ $exposures }[$indexing{$exp_id_B}];
+
+	    $exp_A->{pairwise} = 1;
+	    $exp_A->{private} = 0;
+	    $exp_A->{pair_id} = $exp_B->{chip_id};
+	    
+	    $exp_B->{pairwise} = 1;
+	    $exp_B->{private} = 0;
+	    $exp_B->{pair_id} = $exp_A->{chip_id};
+	    
+	    if ($verbose) {
+		print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n";
+	    }
+	}
+    }
+
+    # Scan all exposures, and ensure that pairwise and private are set correctly
+    foreach my $exposure (@$exposures) { 
+	if ($exposure->{pairwise} && !($exposure->{pair_id})) {
+	    $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error.
+	}
+	if (!($exposure->{pairwise})) {
+	    $exposure->{private} = 1; # If this isn't being pairwise diffed, it needs to be private
+	}
+
+	$exposure = update_this_exposure($exposure);
+    }
+    return(1);
+}
+
+
+#
+# Decide Things
+################################################################################
+sub monitor_mode {
+    my $lap_id = shift;
+    return(check_status($lap_id));
+}
+    
+
+sub check_stack_status {  # look at lapRun associated stacks, and confirm their states.
+    my $lap_id = shift;
+
+    my ($defined_qstack,$have_qstack,$defined_fstack,$have_fstack);
+    
+    my $command = "$laptool -stacks -lap_id $lap_id";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform laptool -stacks: $error_code", $lap_id, "");
+    }
+
+    ($defined_qstack,$have_qstack,$defined_fstack,$have_fstack) = (0,0,0,0);
+    if (@$stdout_buf == 0) {
+	# Nothing to do.  Possibly an error, but for now, just accept it
+	return($defined_qstack,$have_qstack,$defined_fstack,$have_fstack);
+    }
+
+    my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from laptool -stacks", $lap_id, "");
+
+    my $total_qstacks = 0;
+    my $complete_qstacks = 0;
+    my $total_fstacks = 0;
+    my $complete_fstacks = 0;
+    foreach my $stack (@$stacks) {
+	if (($stack->{quick_stack_id})&&
+	    (S64_IS_NOT_NULL($stack->{quick_stack_id}))) {
+	    $defined_qstack = 1;
+	    $total_qstacks++;
+	}
+	if (($stack->{final_stack_id})&&
+	    (S64_IS_NOT_NULL($stack->{final_stack_id}))) {
+	    $defined_fstack = 1;
+	    $total_fstacks++;
+	}
+	
+	if (($stack->{quick_state})&&
+	    ($stack->{quick_state} eq 'full')) {
+	    $complete_qstacks++;
+	}
+	elsif (($stack->{quick_state})&&($stack->{quick_state} eq 'new')&&
+	       ($stack->{quick_fault} >= 4)&&($stack->{quick_fault} != 32767)) {
+	    printf STDERR "Faulted quick stack: $stack->{quick_stack_id} $stack->{quick_fault}\n";
+	    $complete_qstacks++;# This is not the best solution, but if they continually fail, there's not much we can do.
+	}
+	if (($stack->{final_state})&&
+	    ($stack->{final_state} eq 'full')) {
+	    $complete_fstacks++;
+	}
+	elsif (($stack->{final_state})&&($stack->{final_state} eq 'new')&&
+	       ($stack->{final_fault} >= 4)&&($stack->{final_fault} != 32767)) {
+	    printf STDERR "Faulted final stack: $stack->{final_stack_id} $stack->{final_fault}\n";
+	    $complete_fstacks++; # This is not the best solution, but if they continually fail, there's not much we can do.
+	}
+
+    }
+    if (($complete_qstacks > 0)&&
+	($complete_qstacks == $total_qstacks)) {
+	$have_qstack = 1;
+    }
+    if (($complete_fstacks > 0)&&
+	($complete_fstacks == $total_fstacks)) {
+	$have_fstack = 1;
+    }
+    
+	
+    return($defined_qstack,$have_qstack,$defined_fstack,$have_fstack);
+}
+sub check_status {
+    my $lap_id = shift;
+    
+    my ($defined_qstack,$have_qstack,$defined_fstack,$have_fstack) = check_stack_status($lap_id);
+
+    my $command = "$laptool -exposures -lap_id $lap_id";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform laptool -exposures: $error_code", $lap_id);
+    }
+    
+    if (@$stdout_buf == 0) {
+	# Nothing to do. However, this is likely an error.
+	return(0);
+    }
+    
+    my $exposures = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from laptool -exposures", $lap_id);
+
+    # Need to prescan to see who matches whom.
+    my %match_hash = ();
+    my $index = 0;
+    foreach my $exposure (@$exposures) {
+	if ($exposure->{pair_id}) {
+	    $match_hash{$exposure->{pair_id}} = $index; # Tell the companion who we are, so they can find us next time.
+	}
+	$index++;
+    }
+
+    # Things I want to know before I'm through
+    my $needs_qstack = 0;
+    my $needs_something_remade = 0;
+    my $needs_something_private = 0;
+    my $can_qstack = 0;
+    my $have_diff = 0;
+    my $can_diff = 0;
+    my $can_fstack = 0;
+    my $total_exposures = 0;
+    foreach my $exposure (@$exposures) {
+	$total_exposures++;
+	my $companion;
+
+	if ($exposure->{pair_id}) { # Load companion exposure information
+	    if (exists($match_hash{$exposure->{chip_id}})) {
+		$companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; # Match!
+	    }
+	}
+	
+	if ($exposure->{private}) { # I've declared this exposure private to this lapRun.
+	    $needs_qstack = 1;
+	}
+	
+	if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun
+	    $needs_something_remade = 1;
+	    $exposure = remake_this_exposure($exposure);
+	}
+	if ($exposure->{cam_quality}) {
+	    $needs_qstack = 1;
+	    $needs_something_private = 1;
+	    if ($companion) {
+		$companion->{private} = 1;
+		$companion->{pairwise} = 0;
+		&update_this_exposure($companion);
+	    }
+	    $exposure->{private} = 1;
+	    $exposure->{pairwise} = 0;
+	    $exposure->{data_state} = 'drop';
+	    &update_this_exposure($exposure);
+
+	}
+# 	if ($companion) { # Validate that there are no problems with the companion exposure
+# 	    if ($companion->{cam_quality}) { # Maybe other things here?
+# 		$exposure->{private} = 1;
+# 		$exposure->{data_state} = 'drop';
+# 		&update_this_exposure($exposure);
+# 		$needs_qstack = 1;
+# 	    }
+# 	}
+	if  ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through.
+	    $can_qstack ++;
+	    $can_diff ++;
+	    $have_diff ++;
+	    $can_fstack ++;	    
+	    next;
+	}
+
+	if (($exposure->{warpRun_state})&&  # This exposure has a warp
+	    ($exposure->{warpRun_state} eq 'full')) { # This exposure's warp is done.
+	    $can_qstack ++;
+	    $can_diff ++;
+	}
+	if (($exposure->{magicked}&&
+	     &S64_IS_NOT_NULL($exposure->{magicked}))) {  # This exposure has been magicked, so it is through with diff.
+	    $can_fstack ++;
+	}
+	if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) {
+	    $have_diff ++;
+	}
+    }
+
+
+    print "STATUS: DEFINED_QSTACK:  $defined_qstack\n";
+    print "STATUS: HAVE_QSTACK:     $have_qstack\n";
+    print "STATUS: DEFINED_FSTACK:  $defined_fstack\n";
+    print "STATUS: HAVE_FSTACK:     $have_fstack\n";
+
+    print "STATUS: NEEDS_QSTACK:    $needs_qstack\n";
+    print "STATUS: NEEDS_REMADE:    $needs_something_remade\n";
+    print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n";
+    print "STATUS: CAN_QSTACK:      $can_qstack\n";
+    print "STATUS: CAN_DIFF:        $can_diff\n";
+    print "STATUS: HAVE_DIFF:       $have_diff\n";
+    print "STATUS: CAN_FSTACK:      $can_fstack\n";
+
+    print "STATUS: TOTAL_EXPOSURES: $total_exposures\n";
+    if ($do_nothing) {
+	exit(0);
+    }
+    if (($needs_qstack == 1)&&              # Do we need the quick stack?
+	($defined_qstack == 0)&&            # Have we not made it already?
+	($can_qstack == $total_exposures)&& # Are all warps done?
+	($needs_something_remade == 0)      # Do we need to wait for a stolen exposure?
+	) {
+	print "STATUS: Will now queue quickstacks\n";
+	&queue_quickstack($exposures);
+    }
+
+    if (($can_diff == $total_exposures)&&   # Are all warps done?
+	($needs_something_remade == 0)&&    # Do we need to wait for a stolen exposure?
+	($have_diff < $can_diff)&&          # We haven't already done them all.
+	(($needs_qstack == 0)||             # Are we just doing pairwise diffs?
+	 ($needs_qstack == 1)&&($have_qstack == 1)) ) { # If we need a quickstack, do we have it?
+	print "STATUS: Will now queue diffs\n";
+	&queue_diffs($exposures);
+    }
+
+    if (($can_fstack == $total_exposures)&& # Are all warps done?
+	($needs_something_remade == 0)&&    # Do we need to wait for a stolen exposure?
+	($defined_fstack == 0)) {           # Have we not made it already?
+	print "STATUS: Will now queue final stacks\n";
+	&queue_finalstack($exposures);
+    }
+
+    if (($have_fstack == 1)) {              # Are we all done making the final product?
+	print "STATUS: Will now deactivate exposures for this complete lapRun.\n";
+	&deactivate_exposures($exposures);
+	$command = "$laptool -updaterun -lap_id $lap_id";
+	$command .= " -dbname $dbname " if defined $dbname;
+	$command .= " -set_state full ";
+	my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $command, verbose => $verbose);
+	unless ($success) {
+	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	    &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id);
+	}
+    }
+    
+}
+
+sub queue_quickstack {
+    my $exposures = shift; # reference to exposure array;
+    my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information
+    
+    my $lap_id    = $exposure->{lap_id};
+    my $label     = $exposure->{label};
+    my $filter    = $exposure->{filter};
+    my $proj_cell = $exposure->{projection_cell};
+
+    unless (defined($label) && defined($filter) && defined($proj_cell)) {
+	&my_die("Unable to perform stacktool. Insufficient information.", $lap_id);
+    }
+
+    my @utctime = gmtime();
+    $utctime[5] += 1900;
+    $utctime[4] += 1;
+    my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]);
+    my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
+    my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
+    my $data_group = "${label}.${date}";
+
+    my $command = "$stacktool ";
+    $command .= " -pretend " if defined $debug;
+    $command .= " -dbname $dbname " if defined $dbname;
+    $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter ";
+    $command .= " -set_label ${label} -set_data_group ${proj_cell}.quick.${date} ";
+    $command .= "  -set_workdir $workdir  -set_dist_group NODIST ";
+    $command .= " -min_num 2 -set_reduction QUICKSTACK ";
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id);
+    }
+
+    $command = "$stacktool ";
+    $command .= " -dbname $dbname " if defined $dbname;
+    $command .= " -sassskyfile -data_group ${proj_cell}.quick.${date} ";
+    $command .= " -filter $filter -projection_cell ${proj_cell} ";
+
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform stacktool -sassskyfile: $error_code", $lap_id);
+    }
+    
+    my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from stacktool -sassskyfile", $lap_id, "");
+    
+    my $stack = ${ $stacks }[0];
+    my $sass_id = $stack->{sass_id};
+    unless (defined($sass_id)) {
+	&my_die("Unable to parse metadata from stacktool for sass_id", $lap_id, "");
+    }
+
+    print "QUICK_SASS_ID: $sass_id\n";
+    $command = "$laptool -updaterun -lap_id $lap_id -set_quick_sass_id $sass_id";
+    $command .= " -dbname $dbname " if defined $dbname;
+
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform laptool -updaterun: $error_code", $lap_id);
+    }
+    
+
+}
+sub queue_finalstack {
+    my $exposures = shift; # reference to exposure array;
+    my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information
+    
+    my $lap_id    = $exposure->{lap_id};
+    my $label     = $exposure->{label};
+    my $filter    = $exposure->{filter};
+    my $proj_cell = $exposure->{projection_cell};
+
+    unless (defined($label) && defined($filter) && defined($proj_cell)) {
+	&my_die("Unable to perform stacktool. Insufficient information.", $lap_id);
+    }
+
+    my @utctime = gmtime();
+    $utctime[5] += 1900;
+    $utctime[4] += 1;
+    my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]);
+    my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
+    my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
+    my $data_group = "${label}.${date}";
+
+    my $command = "$stacktool ";
+    $command .= " -pretend " if defined $debug;
+    $command .= " -dbname $dbname " if defined $dbname;
+    $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter ";
+    $command .= " -set_label ${label} -set_workdir $workdir -set_data_group ${proj_cell}.final.${date} ";
+    $command .= " -min_num 2 -set_reduction THREEPI_STACK ";
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id);
+    }
+    
+    
+    $command = "$stacktool ";
+    $command .= " -dbname $dbname " if defined $dbname;
+    $command .= " -sassskyfile -data_group ${proj_cell}.final.${date} ";
+    $command .= " -filter $filter -projection_cell ${proj_cell} ";
+
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform stacktool -sassskyfile: $error_code", $lap_id);
+    }
+
+    my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from stacktool -sassskyfile", $lap_id, "");
+
+    my $stack = ${ $stacks }[0];
+    my $sass_id = $stack->{sass_id};
+    unless (defined($sass_id)) {
+	&my_die("Unable to parse metadata from stacktool for sass_id", $lap_id, "");
+    }
+    print "FINAL_SASS_ID: $sass_id\n";
+    $command = "$laptool -updaterun -lap_id $lap_id -set_final_sass_id $sass_id";
+    $command .= " -dbname $dbname " if defined $dbname;
+
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform laptool -updaterun: $error_code", $lap_id);
+    }
+
+}
+sub queue_diffs {
+    my $exposures = shift;
+    
+    # Need to prescan to see who matches whom.
+    my %match_hash = ();
+    my $index = 0;
+    foreach my $exposure (@$exposures) {
+	if ($exposure->{pair_id}) {
+	    $match_hash{$exposure->{pair_id}} = $index; # Tell the companion who we are, so they can find us next time.
+	}
+	$index++;
+    }
+
+    my %already_queued = ();
+    foreach my $exposure (@$exposures) {
+	if ($exposure->{data_state} eq 'drop') { # Kick out unusable exposures
+	    next;
+	}
+	if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # Not sure how this would happen, but still. ## This happens when we inherit a complete exposure.
+	    next;
+	}
+	if ($already_queued{$exposure->{warp_id}}) {
+	    print "STATUS: Have already queued a diff containing $exposure->{exp_id} $exposure->{chip_id} $exposure->{warp_id}\n";
+	    next;
+	}
+
+	my @utctime = gmtime();
+	$utctime[5] += 1900;
+	$utctime[4] += 1;
+
+	my $label = $exposure->{label};
+	my $dist_group = $exposure->{dist_group};
+	my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]);
+	my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
+	my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
+	my $data_group = "${label}.${date}";
+
+
+	my $command = "$difftool ";
+	$command .= " -pretend " if defined $debug;
+	$command .= " -dbname $dbname " if defined $dbname;
+	$command .= " -set_label $label -set_workdir $workdir -set_data_group $data_group ";
+	if ($exposure->{dist_group}) {
+	    $command .= " -set_dist_group $exposure->{dist_group} ";
+	}
+	
+	if ($exposure->{pairwise}) { # warpwarp
+	    my $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}];
+	    $command .= " -definewarpwarp ";
+	    $command .= "-input_label $label -template_label $label -backwards ";
+	    $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} ";
+	    $already_queued{$exposure->{warp_id}} = 1;
+	    $already_queued{$companion->{warp_id}} = 1;
+	    
+	}
+	else { # warp-qstack
+	    $command .= " -definewarpstack -set_reduction NOCONVDIFF -available -good_frac 0.2 ";
+	    $command .= " -warp_id $exposure->{warp_id} -stack_label ${label}.quick ";
+	    $already_queued{$exposure->{warp_id}} = 1;
+	}
+	
+	my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $command, verbose => $verbose);
+	unless ($success) {
+	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	    &my_die("unable to perform difftool -definewarp(warp|stack): $error_code", $exposure->{lap_id}, $exposure->{proj_cell});
+	}
+	
+	my $diffs = $mdcParser->parse_list(join "", @$stdout_buf) or
+	    &my_die("Unable to parse metadata from difftool -definewarp(warp|stack)", $lap_id, "");
+	
+	my $diff = ${ $diffs }[0];
+	my $diff_id = $diff->{diff_id};
+	unless (defined($diff_id)) {
+	    $exposure->{data_state} = 'drop';
+	    &update_this_exposure($exposure);
+	}
+	
+    }
+}
+
+# Deactivate all exposures in this run.
+sub deactivate_exposures {
+    my $exposures = shift;
+    foreach my $exposure (@$exposures) {
+	$exposure->{active} = 0;
+	update_this_exposure($exposure);
+    }
+}
+
+
+#
+# Cleanup
+################################################################################
+
+sub cleanup_mode {
+    my $lap_id = shift;
+    my $command = "$laptool -inactiveexp -lap_id $lap_id ";
+    $command .= " -dbname $dbname " if defined $dbname;
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform laptool -inactiveexp: $error_code", "none", "none");
+    }
+    if (@$stdout_buf == 0) {
+	# Nothing to do.
+	return(0);
+    }
+    
+    my $exposures = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from laptool -inactiveexp", $lap_id);
+
+    my @clean_modes = (
+	'chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -chip_id @CHIP_ID@',
+	'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@',
+	'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@',
+	'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage chip -stage_id @CHIP_ID@',
+	'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage warp -stage_id @WARP_ID@',
+	'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage diff -stage_id @DIFF_ID@');
+    foreach my $exposure (@$exposures) {
+	if ($exposure->{is_in_use}) {
+	    next;
+	}
+ 	foreach my $clean_mode (@clean_modes) {
+ 	    my $command = $clean_mode;
+	    $command =~ s/\@DBNAME\@/$dbname/g;
+	    $command =~ s/\@CHIP_ID\@/$exposure->{chip_id}/;
+	    $command =~ s/\@WARP_ID\@/$exposure->{warp_id}/;
+	    $command =~ s/\@DIFF_ID\@/$exposure->{diff_id}/;
+	    $command =~ s/\@LABEL\@/$exposure->{label}/;
+	    
+	    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform cleantool: $command : $error_code", "none", "none");
+	    }
+ 	}
+ 	$exposure->{data_state} = 'cleaned';
+ 	update_this_exposure($exposure);
+    }
+}
+    
+	    
+
+
+#
+# Utilities
+################################################################################
+
+sub my_die {
+    my $msg = shift; # Warning message on die
+    my $lap_id = shift; # identifier
+    my $optional = shift;
+    my $exit_code = shift;
+    unless (defined $exit_code) {
+	$exit_code = $PS_EXIT_PROG_ERROR;
+    }
+    carp($msg);
+    exit $exit_code;
+}
+
+# Check to see if a 64bit integer is NULL or not.  This is probably fragile, but I don't know how to get a S64 NULL out.
+sub S64_IS_NOT_NULL {
+    if ($_[0] == 9223372036854775807) {
+	return(0);
+    }
+    else {
+	return(1);
+    }
+}
+
+# Take the current exposure structure, and save it to the database.
+sub update_this_exposure {
+    my $exposure = shift;
+    
+    # These are required
+    my $lap_id = $exposure->{lap_id};
+    my $exp_id = $exposure->{exp_id};
+    
+    my $command = "$laptool -updateexp -lap_id $lap_id -exp_id $exp_id ";
+    if (($exposure->{chip_id})&&(S64_IS_NOT_NULL($exposure->{chip_id}))) {
+	$command .= " -set_chip_id  $exposure->{chip_id} ";
+    }
+    if (($exposure->{pair_id})&&(S64_IS_NOT_NULL($exposure->{pair_id}))) {
+	$command .= " -set_pair_id  $exposure->{pair_id} ";
+    }
+
+    if ($exposure->{private}) {
+	$command .= " -private ";
+    }
+    else {
+	$command .= " -public ";
+    }
+    if ($exposure->{pairwise}) {
+	$command .= " -pairwise ";
+    }
+    else {
+	$command .= " -nopairwise ";
+    }
+    if ($exposure->{active}) {
+	$command .= " -active ";
+    }
+    else {
+	$command .= " -inactive ";
+    }
+    if ($exposure->{data_state}) {
+	$command .= " -set_data_state $exposure->{data_state} ";
+    }
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform laptool -updateexp: $error_code", $exposure->{lap_id}, $exposure->{proj_cell});
+    }
+}    
+
Index: trunk/ippScripts/scripts/magic_destreak.pl
===================================================================
--- trunk/ippScripts/scripts/magic_destreak.pl	(revision 31422)
+++ trunk/ippScripts/scripts/magic_destreak.pl	(revision 31435)
@@ -360,5 +360,10 @@
         $command .= " -chip_mask $ch_mask" if defined $ch_mask;
         $command .= " -weight $weight" if defined $weight;
-        $command .= " -sources $sources" if defined $sources;
+	if ((defined($sources))&&($ipprc->file_exists($sources))) {
+	    $command .= " -sources $sources" if defined $sources;
+	}
+	else {
+	    print "Did not add sources because they do not appear to exist. This may be an error.\n";
+	}
         $command .= " -skycelllist $skycell_list" if defined $skycell_list;
         $command .= " -replace" if $replace;