#!/usr/bin/env perl

=pod

=head1 NAME

ingest - Ingest datastore metadata and detections into MOPS

=head1 SYNOPSIS

ingest [options] [EXPNAMES]

  EXPNAMES : staged exposure files to ingest (optional; requires STAGE setup)
  --instance=NAME : specify MOPS instance to ingest into
  --debug : test mode; don't modify DB, don't execute makefiles, dump stuff
  --help : show man page

=head1 DESCRIPTION

The MOPS ingest program obtains a directory of all waiting files in the
IPP outgoing datastore, retrieves these fields, and inserts field metadata
and detections into a MOPS instance.  High-confidence detections are
inserted into the MOPS database, and low-confidence detections are stored
in the simulation's low-confidence archive.  Note that low-confidence
here means all detections with a S/N above the LC threshold, and thus
includes high-confidence detections as well.

=head1 EXIT VALUE

INGEST exits as follows:

  0 - execution finished normally
  99 - did not ingest all fields

=cut

use strict;
use warnings;

use Getopt::Long;
use Pod::Usage;
use Cwd;
use File::Slurp;
use Params::Validate;

use File::Path;
use File::Basename;
use LWP;        # XXX eventually remove

use PS::MOPS::Constants qw(:all);
use PS::MOPS::Lib qw(:all);
use PS::MOPS::DC::Instance;
use PS::MOPS::DC::Field;
use PS::MOPS::DC::Ingestor qw(:all);

use subs qw(
    fetch_fpa_set
);


our $RESULT_SUCCESS = 0;
our $RESULT_NOTHINGTODO = 1;
our $RESULT_FAIL = 99;


my $exitval = 0;        # process exit code; see EXIT VALUE below
my $t0 = time;

my $inst;
my $instance_name;
my $rootdir;
my $debug;
my $help;
GetOptions(
    'instance=s' => \$instance_name,
    'rootdir=s' => \$rootdir,
    debug => \$debug,
    help => \$help
) or pod2usage(2);
pod2usage(-verbose => 3) if $help;
my @ingest_files = @ARGV;       # user-specified files to ingest

$inst = PS::MOPS::DC::Instance->new(DBNAME => $instance_name);
my $mops_config = $inst->getConfig;
my $mops_logger = $inst->getLogger;


# Some global config variables.
my $ds_root_url = $mops_config->{ingest}->{root_url};
my $ds_index_url = $mops_config->{ingest}->{index_url};
my $ipp_clean = $mops_config->{ingest}->{ipp_clean};
my $dedupe = $mops_config->{ingest}->{dedupe};
$rootdir = $mops_config->{ingest}->{rootdir} unless $rootdir;       # get from config unless specified on CL
die "can't determine datastore_host" unless ($ds_root_url or $ds_index_url or $rootdir);
die "No files specified.\n" if ($rootdir and @ingest_files == 0);


# High-confidence S2N cutoff.  There may be synthetic LC data in the
# database, which we'll ignore.
my $s2n_cutoff = $mops_config->{site}->{limiting_s2n} || die "can't get site->limiting_s2n";
my $low_confidence_s2n = $mops_config->{site}->{low_confidence_s2n} || $s2n_cutoff; # default to HC S/N for now

eval {
    if (@ingest_files) {
        process_file($inst, $rootdir, $_) foreach @ingest_files;
    }
    else {
        # We want to finally unify the IPP production and MOPS test datastore interfaces.
        # If the old root_url is specified, rework it to look like $ds_index_url.
        if ($ds_root_url) {
            $ds_root_url =~ s{/$}{};     # strip trailing /
            $ds_index_url = "$ds_root_url/" . $inst->dbname() . '/';
        }

        my $data_format = $mops_config->{ingest}->{data_format};
        if ($data_format =~ /^fpa/) {
            fetch_fpa_set($inst, $ds_index_url);
            # Files must now be deleted outside of MOPS.
        }
        elsif ($data_format eq 'skycell') {
            fetch_skycells($inst, $ds_index_url);
            # Files must now be deleted outside of MOPS.
        }
    }
};

$mops_logger->logdie($@) if $@;                 # log error and bail
$mops_logger->info(
    mopslib_formatTimingMsg(
        subsystem => 'INGEST',
        time_sec => (time - $t0),
        nn => 0,        # no NN information, so punt
    )
);
exit;


sub fetch_fpa_set {
=item fetch_fpa_set

2009-SEP-18 : We have yet another strategy from IPP, and that is 
to ingest full FPA pairs.

* Get the list of new filesets; that is, all filesets newer than our
newest ingest record in the DB.

* For each fileset

  - Download its files into a temporary directory.

  - Insert the .pos and .neg files directly from here.

  - Write an ingest record matching the fileset so that we skip it in
the future.

=cut
    my ($inst, $index_url) = @_;
    my $mops_config = $inst->getConfig();
    my $mops_logger = $inst->getLogger();
    my $var_dir = $inst->getEnvironment('VARDIR');


    my $rv = $RESULT_NOTHINGTODO;

    # Need to do something here if URL is file://...

    # Create our place to put stuff while we're working.
    # Determine our query term, or the oldest ingest item we know about.
    my $query = 'index.txt';

    my $start_file_id = $mops_config->{ingest}->{start_file_id};
    my $end_file_id = $mops_config->{ingest}->{end_file_id};
    my $last_file_id = $mops_config->{ingest}->{last_file_id};

    # Fetch the fileset index from the datastore.
    my $ua = LWP::UserAgent->new;
    my $req = $ua->get($index_url . $query);

    if (!$req->is_success) {
        $mops_logger->logwarn("INGEST: could not read URL $index_url$query" . $req->status_line);
        return $RESULT_FAIL;
    }

    # Turn our results into a list of fileset IDs to retrieve.
    my @filesets = grep { !/^#/ } split /[\r\n]+/, $req->content;

    # See if we are to sort the filesets by ID.  Normally they are sorted in the
    # datastore by time; the IDs are not necessarily in order.
    my @sorted_filesets;
    if ($mops_config->{ingest}->{sort_fileset_ids}) {
        @sorted_filesets = sort @filesets;
    }
    else {
        @sorted_filesets = @filesets;
    }

    # Now see if we are to filter by datastore comment (e.g., to select only sweetspot fields.
    my $filter = $mops_config->{ingest}->{fileset_filter};
    my $fileset_list = $mops_config->{ingest}->{fileset_list};
    if ($fileset_list) {
        my $pat = join('|', map { quotemeta } split /[\|, ]/, $fileset_list);
        @sorted_filesets = grep { /$pat/ } @sorted_filesets;
    }
    elsif ($filter) {
        @sorted_filesets = grep { /$filter/ } @sorted_filesets;
    }

    if ($start_file_id && $end_file_id) {
        # strip the list up to start file ID and after end_file_id
        my @temp;
        my $go = 0;
        foreach (@sorted_filesets) {
            next unless $go or /$start_file_id/;
            push @temp, $_;
            $go = 1;
            last if /$end_file_id/;
        }
        @filesets = @temp;
    }
    elsif ($start_file_id) {
        my @temp;
        my $go = 0;
        foreach (@sorted_filesets) {
            next unless $go or /$start_file_id/;
            push @temp, $_;
            $go = 1;
        }
        @filesets = @temp;
    }
    elsif ($end_file_id) {
        my @temp;
        foreach (@sorted_filesets) {
            push @temp, $_;
            last if /$end_file_id/;
        }
        @filesets = @temp;
    }
    else {
        @filesets = @sorted_filesets;
    }


    if ($mops_config->{ingest}->{reverse_ingest_order}) {
        # Reverse ingest order so that most recent are ingested first.  Then if repeats (older)
        # filesets are encountered, they're skipped.
        @filesets = reverse @filesets;
    }
    
    foreach my $line (@filesets) {
        last if ($end_file_id and $line =~ /$end_file_id/);     # stop ingesting if we've reached this

        $line =~ s/\s+//g;          # strip all whitespace
        my ($fileset_id, $time_registered, $type) = split /\|/, $line;

        $mops_logger->info("INGEST: processing $line");
        process_fpa_set($inst, $mops_config, $mops_logger, "$var_dir/fetch", $ua, $index_url, $fileset_id);

        # The following is a hack for MOPS simulation processing.  If the datastore is local, 
        # remove the files using register_fileset --del.
        if ($index_url =~ /localhost/) {
            my @cmd = (
                'register_fileset',
                '--del',
                '--product=' . $inst->dbname,
                "--fileset=$fileset_id",
            );
            system(@cmd) == 0 or $mops_logger->logdie(@cmd);
        }

        last if ($last_file_id and $line =~ /$last_file_id/);     # stop ingesting if we've reached this
    }

    return $rv;
}


sub process_fpa_set {
    my ($inst, $mops_config, $mops_logger, $tmpdir, $ua, $index_url, $fileset_id) = @_;
    my $req = $ua->get("$index_url/$fileset_id/index.txt");

    unless ($req->is_success) {
        # Anything other than 200 OK.
        $mops_logger->logdie($req->status_line);        # fail for now
    }

    my $rv;
    my $mytmpdir = "$tmpdir/$fileset_id";

    # Set up directories for downloading/processing.
    # First remove if it exists.
    if (-d $mytmpdir) {
        eval { rmtree $mytmpdir };
        if ($@) {
            $mops_logger->logdie("can't rmtree $mytmpdir");
        }
    }
    # Now create a fresh dir.
    eval { mkpath $mytmpdir };
    if ($@) {
        $mops_logger->logdie("can't create $mytmpdir");
    }


    my $t0 = time;
    my @files = grep { !/^#/ } split /[\r\n]+/, $req->content;
    my @retrieved_files;
    foreach my $line (sort { $a cmp $b } @files) {
        $line =~ s/\s+//g;          # strip all whitespace
        my ($file_id, $bytes, $md5sum, $type) = split /\|/, $line;

        if (lc($type) !~ '^ipp-mops') {
            $mops_logger->logwarn('INGEST: unknown file type: ' . $type);
            next;
        }

        if ($bytes <= 0) {
            $mops_logger->logwarn('INGEST: strange IPP file size: ' . $bytes);
            next;
        }

        # Insert into database. 
        $mops_logger->info('INGEST: retrieving FPA ' . join(' ', $fileset_id, $file_id, $bytes));
        my $file_url = "$index_url/$fileset_id/$file_id";
#        $file_url =~ s|/[^/]+$|/$file_id|;           # replace last item with file ID
        my $temp_filename = $mytmpdir . "/$file_id";

        $req = $ua->get($file_url, ':content_file' => $temp_filename);      # fetch catalog from IPP (can be big)

        if (!$req->is_success) {
            $mops_logger->logdie("INGEST: couldn't retrieve FPA file $file_id");      # XXX might want to retry on this one
            return $RESULT_FAIL;
        }
        push @retrieved_files, $temp_filename;
    }

    if (scalar @retrieved_files == 1 and !$mops_config->{ingest}->{accept_singletons}) {
        $mops_logger->info("INGEST: skipping single FPA file $retrieved_files[0]");
        return $rv;
    }

    if (scalar @retrieved_files != 1 and $mops_config->{ingest}->{require_singletons}) {
        $mops_logger->info("INGEST: skipping diff $fileset_id, singletons required");
        return $rv;
    }

    # Do some prefiltering.
    my $wd = getcwd;
    chdir $mytmpdir or die "can't chdir to $mytmpdir";
    my $cmd;
    my $opts = '';
    my @ingest_files = @retrieved_files;


    if ($dedupe) {
        # Remove duplicate detections from file.  Use max density of 1 detection per sq arcsec.
        my $density_perdeg2 = '--density_perdeg2 1';
        my $density_radius_deg = '--density_radius_deg .00027';
        $cmd = "scrubField $density_perdeg2 $density_radius_deg --suffix=dedupe " . join(' ', @ingest_files);
        $mops_logger->info("EXEC $cmd");
        if (0 != system($cmd)) {
            $mops_logger->logdie("$cmd failed");
        }
        @ingest_files = map { "$_.dedupe" } @ingest_files;
    }

    if ($ipp_clean) {
        # Perform other cleaning: spatial density filter to prevent high-density regions.
        my $density_perdeg2 = '--density_perdeg2 ' . ($mops_config->{ingest}->{density_perdeg2} || 50000);
        my $density_radius_deg = '--density_radius_deg ' . ($mops_config->{ingest}->{density_radius_deg} || 0.01);
        $cmd = "scrubField $density_perdeg2 $density_radius_deg --suffix=cleaned " . join(' ', @ingest_files);
        $mops_logger->info("EXEC $cmd");
        if (0 != system($cmd)) {
            $mops_logger->logdie("$cmd failed");
        }
        @ingest_files = map { "$_.cleaned" } @ingest_files;
    }

    # Now ingest the FPA files.
    foreach my $filename (@ingest_files) {
        $mops_logger->info('INGEST: processing FPA ' . $filename);
        insert_ipp_fpa($inst, $filename);
    }


    chdir $wd or die "can't restore $wd";
    $mops_logger->info(
        'INGEST: all FPAs processed in ' 
        . (sprintf "%.1f", time - $t0) 
        . ' seconds.'
    );

    # Remove dir.
    if (!$mops_config->{ingest}->{keep_ingested_files}) {
        eval { rmtree $mytmpdir };
        if ($@) {
            $mops_logger->logdie("can't rmtree $mytmpdir");
        }
    }

    return $rv;
}


sub process_file {
    my ($inst, $rootdir, $file) = @_;
    my $rv;
    my $mytmpdir = File::Temp::tempdir(CLEANUP => 1);


    # Rootdir is a location in the MOPS_STAGE tree that contains the
    # following directory structure:
    # rootdir/
    #   IPP-MOPS/
    #     $OC1
    #       $NN1
    #       $NN2
    #     $OC2
    #       $NN3
    #       $NN4
    #     ...
    # So if our filename looks like an EXP_NAME, map it to an intervening
    # OC and NN, and use that as the directory to load exposures.
    if ($file =~ /^o(\d\d\d\d)\w\d\d\d\d\w$/) {
        my $mjd = 50000 + $1;
        my $nn = mopslib_mjd2nn($mjd, $mops_config->{site}->{gmt_offset_hours});
        my $ocnum = mopslib_mjd2ocnum($mjd);
        $file = "$rootdir/$ocnum/$nn/$file";
    }
    else {
        if ($rootdir && $file !~ m|^/|) {
            $file = "$rootdir/$file";
        }
    }
    $mops_logger->logdie("File $file doesn't exist.\n") unless -f $file;

    my $ingest_file = $file;
    $mops_logger->info('INGEST: processing FPA ' . $ingest_file);
    insert_ipp_fpa($inst, $ingest_file);

    my $t1 = time;
    $mops_logger->info(
        "INGEST: file $file processed in " 
        . (sprintf "%.1f", $t1 - $t0) 
        . ' seconds.'
    );
    $t0 = $t1;

    return $rv;
}



