#!/usr/bin/env perl

use warnings;
use strict;
use Carp;
use IPC::Cmd 0.36 qw( can_run run);
use PS::IPP::Metadata::List qw( parse_md_list );
use PS::IPP::Config 1.01 qw( :standard );
use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
use Pod::Usage qw( pod2usage );
use DateTime;

#
# Set up
################################################################################

my $missing_tools = 0;
my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1);

if ($missing_tools) {
    warn("Can't find required tools.");
    exit($PS_EXIT_CONFIG_ERROR);
}

# my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000');
# my @target_list = ('CMB','M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10',
#                  'STS','SVS','SweetSpot','ThreePi');
# my %tessID_list = ('CMB' => 'RINGS.V0', 'M31' => 'M31', 'MD01' => 'MD01', 'MD02' => 'MD02',
#                  'MD03' => 'MD03', 'MD04' => 'MD04', 'MD05' => 'MD05', 'MD06' => 'MD06',
#                  'MD07' => 'MD07', 'MD08' => 'MD08', 'MD09' => 'MD09', 'MD10' => 'MD10',
#                  'STS' => 'STS', 'SVS' => 'RINGS.V0', 'SweetSpot' => 'RINGS.V0', 'ThreePi' => 'RINGS.V0');
# my %comment_list = ('CMB' => 'CMB_Cold%', 'M31' => 'M31%', 'MD01' => 'MD01%', 'MD02' => 'MD02%',
#                   'MD03' => 'MD03%', 'MD04' => 'MD04%', 'MD05' => 'MD05%', 'MD06' => 'MD06%',
#                   'MD07' => 'MD07%', 'MD08' => 'MD08%', 'MD09' => 'MD09%', 'MD10' => 'MD10%',
#                   'STS' => 'Stellar Transit%', 'SVS' => 'SVS%', 'SweetSpot' => 'Sweetspot%', 'ThreePi' => 'ThreePi%');
# my %stackable_list = ('CMB' => 0, 'M31' => 1, 'MD01' => 1, 'MD02' => 1,
#                   'MD03' => 1, 'MD04' => 1, 'MD05' => 1, 'MD06' => 1,
#                   'MD07' => 1, 'MD08' => 1, 'MD09' => 1, 'MD10' => 1,
#                   'STS' => 1, 'SVS' => 0, 'SweetSpot' => 0, 'ThreePi' => 0);
# my $retention_time = 9000;  # days.


my $db;
my $debug = 0;
my %metadata_out = ();
$metadata_out{nsState} = 'UNDEFINEDERROR';
my $loghead;
chomp($loghead = `date`);
print STDERR 'Starting: ' . $loghead . ' ' . $0 . ' ' . (join ' ', @ARGV) . "\n";

# Grab options
my ( $date, $camera, $dbname, $logfile, $verbose, $manual);
my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only);
my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old);

GetOptions(
    'help|h'               => \$help,
    'date=s'               => \$date, # night to make stacks for
    'camera=s'             => \$camera, # camera
    'dbname=s'             => \$dbname, # Database name
    'logfile=s'            => \$logfile,
    'verbose'              => \$verbose,
    'debug'                => \$debug,
    'isburning'            => \$isburning,
    'force_stack_count'    => \$force_stack_count,
    'test_mode'            => \$test_mode,
    'this_target_only=s'   => \$this_target_only,
    'this_filter_only=s'   => \$this_filter_only,
    'check_registration'   => \$check_registration,
    'define_burntool'      => \$define_burntool,
    'queue_burntool'       => \$queue_burntool,
    'check_chips'          => \$check_chips,
    'queue_chips'          => \$queue_chips,
    'check_stacks'         => \$check_stacks,
    'queue_stacks'         => \$queue_stacks,
    'check_diffs'          => \$check_diffs,
    'queue_diffs'          => \$queue_diffs,
    'clean_old'            => \$clean_old,
    ) or pod2usage ( 2 );
pod2usage( -msg =>
"USAGE: automate_stacks.pl <options...> <mode>
        Options:
           --help                 This help.
           --date YYYY-MM-DD      Work on this date (defaults to today GMT).
           --camera <camera>      Default GPC1.
           --dbname <db>          Default gpc1.
           --verbose
           --isburning            Signal that we are currently burntooling.
           --force_stack_count    Force the chip/warp counts.
           --this_target_only     Process only a single target.
           --this_filter_only     Process only a single filter.
        Modes:
           --check_registration   Confirm the data downloaded correctly.
           --define_burntool      Determine date ranges for burntool.
           --queue_burntool       Issue commands to burntool data.
           --check_chips          Confirm that chips can be built.
           --queue_chips          Issue chiptool commands to queue chips.
           --check_stacks         Confirm that stacks can be built.
           --queue_stacks         Issue stacktool commands to queue stacks.
           --check_diffs          Confirm that diffs can be done.
           --queue_diffs          Issue difftool commands to queue diffs.
           --clean_old            Mark old data 'goto_cleanup'.\n",
           -exitval => 2, ) if (defined($help));
pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
pod2usage(
          -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks",
          -exitval => 3,
          ) unless
    defined $check_registration or defined $define_burntool or defined $queue_burntool or
    defined $queue_chips or defined $queue_stacks or
    defined $check_chips or defined $check_stacks or
    defined $test_mode or defined $clean_old;

# Configurable parameters from our config file.
my @target_list = ();
my @filter_list = ();
my %tessID_list = ();
my %obsmode_list = ();
my %object_list = ();
my %comment_list= ();
my %stackable_list = ();
my %reduction_class = ();
my $retention_time = 9000;

# Grab the configuration data.
my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -";
my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    run(command => $conf_cmd, verbose => $verbose);
unless ($success) {
    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR);
}

my $mdcParser = PS::IPP::Metadata::Config->new;
my $metadata = $mdcParser->parse(join "", @$stdout_buf);
foreach my $entry (@{ $metadata }) {
    if (${ $entry }{name} eq 'RETENTION_TIME') {
        $retention_time = ${ $entry }{value};
    }
    elsif (${ $entry }{name} eq 'FILTERS') {
        push @filter_list, ${ $entry }{value};
    }
    elsif (${ $entry }{name} eq 'TARGETS') {
        my @target_data = @{ ${ $entry }{value} };
        my $this_target = '';
        foreach my $tentry (@target_data) {
            if (${ $tentry }{name} eq 'NAME') {
                $this_target = ${ $tentry }{value};
                push @target_list, $this_target;
            }
            elsif (${ $tentry }{name} eq 'TESS') {
                $tessID_list{$this_target} = ${ $tentry }{value};
            }
            elsif (${ $tentry }{name} eq 'OBSMODE') {
                $obsmode_list{$this_target} = ${ $tentry }{value};
            }
            elsif (${ $tentry }{name} eq 'OBJECT') {
                $object_list{$this_target} = ${ $tentry }{value};
            }
            elsif (${ $tentry }{name} eq 'COMMENT') {
                $comment_list{$this_target} = ${ $tentry }{value};
            }
            elsif (${ $tentry }{name} eq 'STACKABLE') {
                $stackable_list{$this_target} = ${ $tentry }{value};
            }
	    elsif (${ $tentry }{name} eq 'REDUCTION') {
		$reduction_class{$this_target} = ${ $tentry }{value};
	    }
        }
    }
}




unless(defined($date)) {
    my @trash = gmtime;
    $trash[5] += 1900;
    $trash[4] += 1;
    $date = sprintf("%4d-%02d-%02d",$trash[5],$trash[4],$trash[3]);
}
unless(defined($camera)) {
    $camera = 'GPC1';
}
unless(defined($dbname)) {
    $dbname = 'gpc1';
}

if (defined($this_target_only)) {
    foreach my $t (@target_list) {
        if ($t eq $this_target_only) {
            @target_list = ($this_target_only);
            last;
        }
    }
    die("$this_target_only is invalid.") if ($#target_list != 0);
}

if (defined($this_filter_only)) {
    foreach my $t (@filter_list) {
        if ($t eq $this_filter_only) {
            @filter_list = ($this_filter_only);
            last;
        }
    }
    die("$this_filter_only is invalid.") if ($#filter_list != 0);
}

#
# Mode selection
################################################################################
if (defined($test_mode)) {
    $debug = 1;
}

if (defined($check_registration) || defined($test_mode)) {
    $metadata_out{nsState} = 'NEW';
    my ($Nsummit_exp,$Nfaults) = check_summit_copy($date);
    return_metadata($date);
    unless (defined($test_mode)) { exit(0); }
}
if (defined($define_burntool) || defined($test_mode)) {
    $metadata_out{nsState} = 'QUEUEBURNING';
    &find_burntool_ranges($date);
    return_metadata($date);
    unless (defined($test_mode)) { exit(0); }
}
if (defined($queue_burntool) || defined($test_mode)) {
    $metadata_out{nsState} = 'BURNING';
    return_metadata($date);
    unless (defined($test_mode)) { exit(0); }
}
if (defined($check_chips) || defined($test_mode)) {
    $metadata_out{nsState} = 'QUEUECHIPS';
    &execute_chips($date,"pretend");
    if (($metadata_out{nsState} eq 'NEEDSBURNING')&&(defined($isburning))) {
        $metadata_out{nsState} = 'BURNING';
    }
    return_metadata($date);
    unless (defined($test_mode)) { exit(0); }
}
if (defined($queue_chips)) {
    $metadata_out{nsState} = 'TOWARP';
    &execute_chips($date);
    return_metadata($date);
    exit(0);
}
if (defined($check_stacks) || defined($test_mode)) {
    $metadata_out{nsState} = 'TOWARP';
    &execute_stacks($date,"pretend");
    if ($metadata_out{nsState} eq 'FORCETOWARP') {
        $metadata_out{nsState} = 'TOWARP';
    }
    return_metadata($date);
    unless (defined($test_mode)) { exit(0); }
}
if (defined($queue_stacks)) {
    $metadata_out{nsState} = 'STACKING';
    &execute_stacks($date);
    if ($metadata_out{nsState} == 'QUEUESTACKING') {
	$metadata_out{nsState} = 'STACKING_POSSIBLE_ERROR';
    }
    return_metadata($date);
    exit(0);
}
if (defined($check_diffs) || defined($queue_diffs)) {
    die("Diffs are currently not implemented.");
}
if (defined($clean_old) || defined($test_mode)) {
    if (defined($test_mode)) {
        &execute_cleans($date,"pretend");
    }
    else {
        &execute_cleans($date);
    }
    unless (defined($test_mode)) { exit(0); }
}
exit(0);
#
# Registration
################################################################################

sub check_summit_copy {
    my $date = shift;
    my $db = init_gpc_db();

    # largely stolen from Bill's checkexp program.

    my $sth = " SELECT exp_name, summitExp.dateobs AS registered, rawExp.dateobs, summitExp.imfiles, ";
    $sth .= " summitExp.fault AS summit_fault, pzDownloadExp.state AS download_state, ";
    $sth .= " count(pzDownloadImfile.class_id) AS download_count, newExp.state AS newExp_state, newExp.exp_id, summitExp.exp_type";
    $sth .= " FROM summitExp LEFT JOIN pzDownloadExp USING(exp_name) LEFT JOIN pzDownloadImfile USING(exp_name) ";
    $sth .= " LEFT JOIN newExp ON exp_name = tmp_exp_name LEFT JOIN rawExp USING(exp_id, exp_name) ";
    $sth .= " WHERE date(summitExp.dateobs) >= '${date}T00:00:00' AND date(summitExp.dateobs) <= '${date}T23:59:59' ";
    $sth .= " GROUP BY exp_name ORDER BY summitExp.dateobs ";

    my $data_ref = $db->selectall_arrayref( $sth );

    my $Nsummit_exps = 0;
    my $Nsummit_faults = 0;
    my $Ndownload_faults = 0;
    my $Nregister_faults = 0;

#    my_trace($sth,$data_ref,$#{ $data_ref });

    foreach my $row_ref (@{ $data_ref }) {
        my ($exp_name,$registered,$dateobs,$imfiles,$summit_fault,
            $download_state,$download_count,$new_state,$exp_id,$exp_type) = @{ $row_ref };
        $Nsummit_exps++;
        if ($summit_fault) {
	    print STDERR "check_summit_copy: $date $exp_name has summit_fault $summit_fault";
            if ($exp_type ne 'OBJECT') {
                print STDERR " (but I don't care).\n";
            }
            else {
                print STDERR "\n";
                $Nsummit_faults++;
            }
        }
        elsif (!$download_state or $download_state eq 'run') {
            print STDERR "check_summit_copy: $date $exp_name has download_state $download_state";
            if ($exp_type ne 'OBJECT') {
                print STDERR " (but I don't care).\n";
            }
            else {
                print STDERR "\n";
                $Ndownload_faults++;
            }
        }
        elsif (!$new_state or $new_state eq 'run' ) {
            print STDERR "check_summit_copy: $date $exp_name has new_state $new_state";
            if ($exp_type ne 'OBJECT') {
                print STDERR " (but I don't care).\n";
            }
            else {
                print STDERR "\n";
                $Nregister_faults++;
            }
        }
    }

    my $Nfaults = $Nsummit_faults + $Ndownload_faults + $Nregister_faults;
    if ($Nsummit_exps == 0) {
        print STDERR "No exposures were found on the summit for $date.\n";
        $metadata_out{nsState} = 'DROP';
    }
    elsif ($Nfaults != 0) {
        print STDERR "There were faults while downloading the exposures for $date.\n";
        $metadata_out{nsState} = 'NEW';
    }
    else {
        print STDERR "Summit copy and Registration have succeeded for $date.\n";
        $metadata_out{nsState} = 'REGISTERED';
    }

    return($Nsummit_exps,$Nfaults);
}

#
# Burntool
################################################################################

sub get_goodBTvalue {
    my $mdcParser = PS::IPP::Metadata::Config->new;

    my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL.STATE.GOOD | uniq";
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run ( command => $config_cmd, verbose => $verbose);
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, 0, $PS_EXIT_SYS_ERROR);
    }
    my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
        &my_die("Unable to parse metadata config doc", 0, 0, 0, $PS_EXIT_SYS_ERROR);
    my $burntoolStateGood = 999;
    foreach my $cfg (@$recipeData) {
        if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') {
            $burntoolStateGood = $cfg->{value};
        }
    }
    if ($burntoolStateGood == 999) {
        &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, 0, 0, $PS_EXIT_SYS_ERROR);
    }
    return($burntoolStateGood);
}

sub verify_burntool {
    my $exp_id = shift;
    my $burntoolStateGood = shift;

    my $db = init_gpc_db();

    my $sth = "SELECT exp_id,exp_name,obs_mode,dateobs,class_id,burntool_state,comment FROM rawImfile WHERE exp_id = $exp_id";
    my $data_ref = $db->selectall_arrayref( $sth );

    my $Nimfiles = 0;
    my $Nburntooled = 0;

    foreach my $row_ref (@{ $data_ref }) {
        my ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$burntool_state,$comment) = @{ $row_ref };
        $Nimfiles++;
        if (abs($burntool_state) == $burntoolStateGood) {
            $Nburntooled++;
        }
    }
    return($Nimfiles,$Nburntooled);
}

sub find_burntool_ranges {
    my $date = shift;
    # Much cleaner than reimplementing that mess here
    my $command = "$mkBTpcontrol -d $date -b";
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run ( command => $command, verbose => $verbose );
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform $command: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
    }

    if ($#{ $stderr_buf } > -1) {
        $metadata_out{nsState} = 'ERROR';
        return(1);
    }

    my $N = 0;
    my @burntool_entries = split /\n/, (join '', @$stdout_buf);
    foreach my $row (@burntool_entries) {
        my ($trash,$start,$end);
        ($trash,$trash,$start,$end) = (split /\s+/,$row);

        for (my $class_counter = 0; $class_counter < 60; $class_counter++) {
            $metadata_out{"bt${N}Begin"} = $start;
            $metadata_out{"bt${N}End"} = $end;
            $metadata_out{"bt${N}Class"} = $class_counter;
            $metadata_out{"bt${N}Status"} = 'NEW';
            $N++;
        }
        print STDERR "define_burntool: $row\n";
    }
    $metadata_out{btN} = $N - 1;
    $metadata_out{btNCounter} = 0;

    return(0);
}

#
# Chips
################################################################################

sub construct_chiptool_cmd {
    my $date = shift;
    my $target = shift;

    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);

    my $select =  "-dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 ";
    $date =~ s/-//g;

    my $cmd = "$chiptool";
    $cmd .= ' -simple -dbname gpc1 -definebyquery -set_end_stage warp ';
    $cmd .= " -set_label $label ";
    $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
    $cmd .= " -set_tess_id $tess_id -set_data_group $data_group ";
    if (defined($obs_mode)) {
        $cmd .= " -obs_mode '$obs_mode' ";
    }
    if (defined($object)) {
        $cmd .= " -object '$object' ";
    }
    if (defined($comment)) {
        $cmd .= " -comment '$comment' ";
    }
    if (defined($reduction)) {
	$cmd .= " -set_reduction $reduction ";
    }
    $cmd .= " $select ";
    if ($debug == 1) {
        $cmd .= " -pretend ";
    }
    print STDERR "$cmd\n";
    return($cmd);
}

sub verify_uniqueness_chip {
    my $exp_id = shift;
    my $date = shift;
    my $target = shift;

    my $db = init_gpc_db();
    $date =~ s/-//g;
    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);

    my $sth = "SELECT exp_id from chipRun where data_group = '$data_group' AND exp_id = $exp_id";
    my $data_ref = $db->selectall_arrayref( $sth );

    return($#{ $data_ref } + 1);
}

sub pre_chip_queue {
    my $date = shift;
    my $target = shift;

    my $command = construct_chiptool_cmd($date,$target) . ' -pretend ';
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run ( command => $command, verbose => $verbose );
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
    }

    my $burntoolStateGood = get_goodBTvalue();
    my $Nimfiles = 0;
    my $Nburntooled = 0;
    my $Nalready = 0;
    my @input_exposures = split /\n/, (join '', @$stdout_buf);
    foreach my $entry (@input_exposures) {
        my ($exp_id, @trash) = split /\s+/, $entry;
        @trash = verify_burntool($exp_id,$burntoolStateGood);
        $Nimfiles += $trash[0];
        $Nburntooled += $trash[1];

        @trash = verify_uniqueness_chip($exp_id,$date,$target);
        $Nalready += $trash[0];
    }
    return($#input_exposures + 1,$Nimfiles,$Nburntooled,$Nalready);
}

sub chip_queue {
    my $date = shift;
    my $target = shift;

    my $command = construct_chiptool_cmd($date,$target);
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run ( command => $command, verbose => $verbose );
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
    }
    $metadata_out{nsState} = 'TOWARP';
    return(0);
}

sub execute_chips {
    my $date = shift;
    my $pretend = shift;
    my $exposures = 0;
    foreach my $target (@target_list) {
        my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = pre_chip_queue($date,$target);
        if ($Nexposures == 0) {
	    print STDERR "execute_chips: Target $target on $date had no exposures.\n";
	    next;
        }
        if ($Nalready != 0) {
	    print STDERR "execute_chips: Not queueing $target on $date due to already existing exposures.\n";
            next;
        }
        if ($Nimfiles != $Nburntooled) {
	    print STDERR "execute_chips: Target $target on $date is not fully burntooled.\n";
            $metadata_out{nsState} = 'NEEDSBURNING';
	    $exposures++;
            next;
        }
	$exposures++;
        unless(defined($pretend)) {
            chip_queue($date,$target);
        }
    }
    if ($exposures == 0) {
	$metadata_out{nsState} = 'DROP';
    }
}

#
# Stacking
################################################################################

sub construct_stacktool_cmd {
    my $date = shift;
    my $target = shift;
    my $filter = shift;

    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);

    my $select =  "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 ";
    $date =~ s/-//g;

    my $cmd = "$stacktool";
#    $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 ';  # Probably silly, but I want to be safe and not overwrite
    $cmd .= ' -simple -dbname gpc1 -definebyquery ';
    $cmd .= " -set_label $label -select_label $label ";
    $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
    $cmd .= " -select_filter $filter -set_data_group $data_group ";
    $cmd .= " -min_num 4";
    $cmd .= " -select_good_frac_min 0.05";
    $cmd .= " $select ";
    if ($debug == 1) {
        $cmd .= ' -pretend ';
    }
    print STDERR "$cmd\n";
    return($cmd);
}

sub verify_uniqueness_stack {
    my $skycell = shift;
    my $date = shift;
    my $target = shift;
    my $filter = shift;

    my $db = init_gpc_db();
    $date =~ s/-//g;
    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);

    my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter'";
    my $data_ref = $db->selectall_arrayref( $sth );

    return($#{ $data_ref } + 1);
}

sub pre_stack_queue {
    my $date = shift;
    my $target = shift;
    my $filter = shift;

    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
    # check warp stage == chip stage
    my $db = init_gpc_db();

    my $trunc_date = $date; $trunc_date =~ s/-//g;

    my $where = " label = '$label' AND data_group = '$data_group' ";
    my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where ";
    my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where ";

    my $chip_ref = $db->selectall_arrayref( $chip_sth );
    my $warp_ref = $db->selectall_arrayref( $warp_sth );

    # check that we will be able to queue them up
    my $command = construct_stacktool_cmd($date,$target,$filter) . ' -pretend ';
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run ( command => $command, verbose => $verbose );
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform stacktool: $error_code", $#{ $chip_ref },$#{ $warp_ref },$date, $PS_EXIT_SYS_ERROR);
    }

    my $Nalready = 0;

    my @input_exposures = split /\n/, (join '', @$stdout_buf);
    foreach my $entry (@input_exposures) {
        my ($warp_tess_id,$skycell, @trash) = split /\s+/, $entry;
        @trash = verify_uniqueness_stack($skycell,$date,$target,$filter);
        $Nalready += $trash[0];
    }

    return($#input_exposures  + 1, $#{ $chip_ref } + 1, $#{ $warp_ref } + 1, $Nalready);
}

sub stack_queue {
    my $date = shift;
    my $target = shift;
    my $filter = shift;

    my $command = construct_stacktool_cmd($date,$target,$filter);
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run ( command => $command, verbose => $verbose );
    unless ($success) {
        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
        &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
    }
    return(0);
}

sub execute_stacks {
    my $date = shift;
    my $pretend = shift;
    foreach my $target (@target_list) {
        if ($stackable_list{$target} == 1) {
            foreach my $filter (@filter_list) {
                my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
                if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
		    if ($debug == 1) {
			print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
		    }
                    $metadata_out{nsState} = 'FORCETOWARP';
                    next;
                }
                if ($Nexposures == 0) {
		    if ($debug == 1) {
			print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n";
		    }
                    next;
                }
                if ($Nalready != 0) {
		    if ($debug == 1) {
			print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
		    }
		    unless ($metadata_out{nsState} eq 'FORCETOWARP') {
			$metadata_out{nsState} = 'STACKING';
		    }
                    next;
                }
                unless (($metadata_out{nsState} eq 'FORCETOWARP')||($metadata_out{nsState} eq 'STACKING')) {
                    $metadata_out{nsState} = 'QUEUESTACKS';
                }
                unless(defined($pretend)) {
		    if ($debug == 1) {
			print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n";
		    }
                    stack_queue($date,$target,$filter);
                }
            }
        }
        else {
            # print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
        }
    }

}

#
# Auto-Clean
################################################################################

sub construct_cleantool_args {
    my $date = shift;
    my $target = shift;

    my ($year,$month,$day) = split /-/,$date;
    my $dt = DateTime->new(year => $year, month => $month, day => $day,
                               hour => 0, minute => 0, second => 0, nanosecond => 0,
                               time_zone => 'Pacific/Honolulu');
    $dt->subtract(days => $retention_time);
    my $cleaning_date = $dt->ymd;

    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target);

    my $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -data_group $data_group ";
    if ($debug == 1) {
        $args .= ' -pretend ';
    }
    return($cleaning_date,$args);
}

sub execute_cleans {
    my $date = shift;
    my $pretend = shift;

    foreach my $target (@target_list) {
        my ($cleaning_date,$args) = construct_cleantool_args($date,$target);

        my $command = $chiptool . $args;
        print STDERR "$command\n";
        if (!(defined($pretend) || $debug == 1)) {
#           print STDERR "BEAR IS DRIVING!?\n";
            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                run ( command => $command, verbose => $verbose );
            unless ($success) {
                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
            }
        }
        $command = $warptool . $args;
        print STDERR "$command\n";
        if (!(defined($pretend) || $debug == 1)) {
#           print STDERR "BEAR IS DRIVING $pretend $debug!?\n";
            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                run ( command => $command, verbose => $verbose );
            unless ($success) {
                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
            }
        }
    }
    return(0);
}




#
# Utilities
################################################################################

sub get_tool_parameters {
    my $date = shift;
    my $target = shift;
    my $workdir_date = $date; $workdir_date =~ s%-%/%g;
    my $trunc_date = $date; $trunc_date =~ s/-//g;

    my $label = "${target}.nightlyscience";
    my $workdir = "neb://\@HOST\@.0/gpc1/${target}.nt/${workdir_date}";
    my $obs_mode = $obsmode_list{$target};
    my $object   = $object_list{$target};
    my $comment = $comment_list{$target};
    my $dist_group = $target;
    my $data_group = "${target}.${trunc_date}";
    my $tess_id = $tessID_list{$target};
    my $reduction = $reduction_class{$target};
    return($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction);
}


sub init_gpc_db {
    use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
    my $dbserver = 'ippdb01';
    my $dbuser = 'ippuser';
    my $dbpass = 'ippuser';
    $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" .
                       "mysql_socket=" . DB_SOCKET(),
                       ${dbuser},${dbpass},
                       { RaiseError => 1, AutoCommit => 1}
        ) or die "Unable to connect to database $DBI::errstr\n";
    return($db);
}

sub return_metadata {
    my $date = shift;
    print STDOUT "autoStack MULTI\n\n";
    print STDOUT "autoStack METADATA\n";
    print STDOUT "   date                  STR          $date\n";
    foreach my $k (sort(keys %metadata_out)) {
        print STDOUT "   $k                STR          $metadata_out{$k}\n";
    }
    print STDOUT "END\n";
}

sub my_die {
    my $msg = shift; # Warning message on die
    my $stage = shift; # stage name
    my $stage_id = shift; #  identifier
    my $exit_code = shift; # Exit code
    # outputImage and path_base are globals

    carp($msg);
    exit $exit_code;
}

sub my_trace {
    if ($debug == 1) {
        foreach my $thing (@_) {
            carp($thing);
        }
    }
}
