#!/usr/bin/env perl

=pod

=head1 NAME

fetchSMFPairs - Query IPP datastore and fetch SMF exposure pairs

=head1 SYNOPSIS

fetchSMFPairs [options]

  --instance=NAME : specify MOPS instance to ingest into
  --nofetch : don't actually fetch, just report what would be fetched
  --debug : test mode; don't modify DB, don't execute makefiles, dump stuff
  --limit N : only ingest this many pairs
  --help : show man page

=head1 DESCRIPTION


=head1 EXIT VALUE

INGEST exits as follows:

  0 - execution finished normally
  99 - did not ingest all fields

=cut

use strict;
use warnings;

use Getopt::Long;
use Pod::Usage;
use Cwd;
use File::Temp;
use Params::Validate;
use LWP;        # XXX eventually remove

use PS::MOPS::Constants qw(:all);
use PS::MOPS::Lib qw(:all);
use PS::MOPS::DC::Instance;
use PS::MOPS::DC::Field;
use PS::MOPS::DC::IngestItem;
use PS::MOPS::DC::Ingestor qw(:all);
use PS::MOPS::IPPDB;


our $RESULT_SUCCESS = 0;
our $RESULT_FAIL = 99;


my $exitval = 0;        # process exit code; see EXIT VALUE below
my $t0 = time;

my $inst;
my $instance_name;
my $nofetch;
my $debug = 0;
my $limit;
my $help;
GetOptions(
    'instance=s' => \$instance_name,
    nofetch => \$nofetch,
    debug => \$debug,
    'limit=n' => \$limit,
    help => \$help
) or pod2usage(2);
pod2usage(-verbose => 3) if $help;

$inst = PS::MOPS::DC::Instance->new(DBNAME => $instance_name);
my $mops_config = $inst->getConfig;
my $tracklet_config = $mops_config->{tracklet};
my $ingest_config = $mops_config->{ingest};
my $mops_logger = $inst->getLogger;
my $var_dir = $inst->getEnvironment('VARDIR');


# Some global config variables.
my $ds_root_url = $ingest_config->{root_url};
my $ds_index_url = $ingest_config->{index_url};
my $ipp_clean = $ingest_config->{ipp_clean};
die "can't determine datastore_host" unless ($ds_root_url or $ds_index_url);

if ($ingest_config->{data_format} !~ /^smf_pairs/) {
    die "ingest->data_format must be 'smf_pairs'";  # sanity check
}

my @all_filesets = fetch_filesets();

# Map fileset names to exp_names.
my %expname2fsn_map;
my $exp_name;
foreach my $fsn (@all_filesets) {
    $fsn =~ s/\|.*$//;  # strip everything after first | to get fileset ID

    # Try various patterns to suss out the exposure name from the fileset ID.
    ($exp_name) = $fsn =~ /^(o\w+)\./;
    unless ($exp_name) {
        ($exp_name) = $fsn =~ /^(o\w+)\./;
    }

    next unless $exp_name;                  # give up

    $fsn =~ s/^\s+|\s+$//g;     # strip
    $expname2fsn_map{$exp_name} = $fsn;
}

my $patch_data = PS::MOPS::IPPDB::patch_exposures(keys %expname2fsn_map);

my @filtered_fields;
my $comment_re;
my $comment_str;
my $ocnum_re;
my $ocnum_str;

$comment_str = $mops_config->{ingest}->{comment_filter};
if ($comment_str) {
    $comment_re = qr/$comment_str/;
}

$ocnum_str = $mops_config->{ingest}->{ocnum_filter};
if ($ocnum_str) {
    $ocnum_re = qr/$ocnum_str/;
}

my $v;
foreach my $k (keys %{$patch_data}) {
    $v = $patch_data->{$k};
    next unless (!$comment_re or $v->{rawData}->{comment} =~ $comment_re);
    next unless (!$ocnum_re or $v->{ocnum} =~ $ocnum_re);
    push @filtered_fields, $v;
}

# Finally we have our list of filesets to process.  Assemble them into pairs so we can
# do pairwise stationary removal.
my $tuple_data = mopslib_assembleTTITuples(
    fields => \@filtered_fields,
    min_fields => $tracklet_config->{tuplewise_min_fields},
    nominal_tti_min => $tracklet_config->{tti_min},
);

# For each tuple, download, remove stationaries and insert into DB.
my $tmap = $tuple_data->{TTI_TUPLES};
foreach my $tkey (sort keys %$tmap) {
    my $tuple = $tmap->{$tkey};     # list of exp_names
    if (scalar @{$tuple} != 2) {
        $mops_logger->logdie("tuple size is not 2: $tkey");
    }

    # Find the corresponding datastore entires.
    my ($tfs1, $tfs2) = @expname2fsn_map{($tuple->[0]->{fieldId}, $tuple->[1]->{fieldId})};
    ingest_smf_pair($inst, $patch_data, $tfs1, $tfs2, $tuple->[0]->{fpaId}, $tuple->[1]->{fpaId});
    last if defined($limit) and --$limit == 0;
}

$mops_logger->logdie($@) if $@;                 # log error and bail
$mops_logger->info(
    mopslib_formatTimingMsg(
        subsystem => 'INGEST',
        time_sec => (time - $t0),
        nn => 0,        # no NN information, so punt
    )
);
exit;


sub fetch_filesets {
    # Create our place to put stuff while we're working.
    # Determine our query term, or the oldest ingest item we know about.
    my $query = 'index.txt';
    my $oldest = PS::MOPS::DC::IngestItem::fetch_oldest($inst);
    if ($oldest) {
        $query .= '?' . $oldest->fileId;
    }

    my $start_file_id = $ingest_config->{start_file_id};
    my $end_file_id = $ingest_config->{end_file_id};
    my $index_url = $ingest_config->{index_url};

    # Fetch the fileset index from the datastore.
    my $ua = LWP::UserAgent->new;
    my $req = $ua->get($index_url . $query);

    if (!$req->is_success) {
        $mops_logger->logwarn("INGEST: could not read URL $index_url$query" . $req->status_line);
        return $RESULT_FAIL;
    }

    my @filesets;

    # Turn our results into a list of fileset IDs to retrieve.
    @filesets = grep { !/^#/ } split /[\r\n]+/, $req->content;

    # Now see if we are to filter by datastore comment (e.g., to select only sweetspot fields.
    my $filter = $ingest_config->{fileset_filter};
    if ($filter) {
        @filesets = grep { /$filter/ } @filesets;
    }

    if (!$oldest) {
        if ($start_file_id && $end_file_id) {
            # strip the list up to start file ID and after end_file_id
            my @temp;
            my $go = 0;
            foreach (@filesets) {
                next unless $go or /$start_file_id/;
                push @temp, $_;
                $go = 1;
                last if /$end_file_id/;
            }
            @filesets = @temp;
        }
        elsif ($start_file_id) {
            my @temp;
            my $go = 0;
            foreach (@filesets) {
                next unless $go or /$start_file_id/;
                push @temp, $_;
                $go = 1;
            }
            @filesets = @temp;
        }
        elsif ($end_file_id) {
            my @temp;
            foreach (@filesets) {
                push @temp, $_;
                last if /$end_file_id/;
            }
            @filesets = @temp;
        }
    }

    return @filesets;
}


sub ingest_smf_pair {
    my ($inst, $patch_data, $tfs1, $tfs2, $fpa_id1, $fpa_id2) = @_;
    my ($f1) = modcf_retrieve($inst, fpaId => $fpa_id1);
    my ($f2) = modcf_retrieve($inst, fpaId => $fpa_id2);

    # First check if we've ingested both of the fields already.  If so, skip.
    if ($f1 and $f2) {
        printf STDERR "Already ingested set $fpa_id1 and $fpa_id2.\n" if $debug;
        return;
    }
    print STDERR "Fetching $tfs1 $tfs2 [", 
        join(' ', 
            $patch_data->{$fpa_id1}->{rawData}->{comment},
            $patch_data->{$fpa_id2}->{rawData}->{comment},
        ), "]\n";

    # Create a temporary work area for each fileset pair.
    my $tmpdir = File::Temp::tempdir('/tmp/smfXXXXXXXX', CLEANUP => 1)
        or $mops_logger->logdie("can't create temp dir");

    my $oldwd = getcwd();
    chdir $tmpdir or $mops_logger->logdie("can't chdir to $tmpdir");

    unless ($nofetch) {
        system('ingestSMFPair', $tfs1, $tfs2) == 0 or die $!;
    }

    chdir $oldwd or die "can't restore old wd $oldwd";
    File::Temp::cleanup();      # force cleanup of temp dir
}
