#!/usr/bin/env perl
#
# Data Store file set registration and de-registration program
#

use strict;
use warnings;

use Getopt::Long qw( GetOptions );
use Pod::Usage qw( pod2usage );
use Digest::MD5::File qw( file_md5_hex );
use DBI;
#use DBD::mysql 4.005;  # needed to set this one day on ipp000
use PS::IPP::Metadata::Config;
use PS::IPP::Metadata::Stats;
use PS::IPP::Metadata::List qw( parse_md_list );
use File::Copy;

use PS::IPP::Config qw($PS_EXIT_SUCCESS
                       $PS_EXIT_UNKNOWN_ERROR
                       $PS_EXIT_SYS_ERROR
                       $PS_EXIT_CONFIG_ERROR
                       $PS_EXIT_PROG_ERROR
                       $PS_EXIT_DATA_ERROR
                       $PS_EXIT_TIMEOUT_ERROR
                       metadataLookupStr
                       metadataLookupBool
                       caturi
                       );
my $product;
my $fileset;
my $filelist;

my $fstype;
my ($ps0, $ps1, $ps2, $ps3, $ps4, $ps5, $ps6, $ps7);

my $linkfiles;      # if set, put links to files in datapath in the Data Store
my $copyfiles;      # if set, copy files from this directory to the Data Store fileset
my $datapath;       # source for files required if $linkfiles or $copyfiles

my $dbname;                     # Database name
my $dsroot;

my $add;
my $del;
my $remove;

my $verbosity = 1;
my $no_cleanup = 1; # if something goes wrong don't delete copied or linked files for debug

#
# parse args
#

GetOptions(
        'add'           =>      \$add,
        'del'           =>      \$del,
        'product=s'     =>      \$product,
        'fileset=s'     =>      \$fileset,
        'list=s'        =>      \$filelist,
        'type=s'        =>      \$fstype,
        'datapath=s'    =>      \$datapath,
        'link'          =>      \$linkfiles,
        'copy'          =>      \$copyfiles,
        'rm'            =>      \$remove,
        'ps0=s'         =>      \$ps0,          # product specific columns 0 - 7
        'ps1=s'         =>      \$ps1,
        'ps2=s'         =>      \$ps2,
        'ps3=s'         =>      \$ps3,
        'ps4=s'         =>      \$ps4,
        'ps5=s'         =>      \$ps5,
        'ps6=s'         =>      \$ps6,
        'ps7=s'         =>      \$ps7,
        'dbname=s'      =>      \$dbname,
        'dsroot=s'      =>      \$dsroot,
) or pod2usage(2);

my $err = "";

$err .= "--product is required\n" unless defined $product;
$err .= "--fileset is required\n" unless defined $fileset;


# will exit if neither or both -add and -del were specified
$err .= "Must specify either --add or --del.\n"
    unless $add xor $del;

if ($add) {
    $err .= "--type is required\n" unless defined $fstype;
    $err .= "--list is required\n" unless defined $filelist;
    if ($linkfiles and $copyfiles) {
        $err .= "only one of --link and --copy is allowed\n";
    }
    if (($linkfiles or $copyfiles) and !$datapath) {
        $err .= "need to specify datapath with --link or --copy\n";
    }
    if ($linkfiles && (substr($datapath, 0, 1) ne "/")) {
        $err .= "datapath must begin with / when using --link\n";
    }
}


#
# lookup the location of the Data Store
#

my $ipprc =  PS::IPP::Config->new(); # IPP Configuration
my $siteConfig = $ipprc->{_siteConfig};
if (!$dsroot) {
    $dsroot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');
    if (!$dsroot) {
        die("Data Store root directory not set");
    }
}


if (!stat("$dsroot/index.txt")) {
    $err .= "Data Store not found at '$dsroot'.\n"
}

# bail if any of the above args were improper
show_usage($err)
    if ($err);

show_usage("Invalid product '$product'.")
    unless defined stat("$dsroot/$product/index.txt");

my $dbserver = metadataLookupStr($siteConfig, 'DBSERVER');
$dbname      = metadataLookupStr($siteConfig, 'DBNAME') unless defined $dbname;
my $dbuser   = metadataLookupStr($siteConfig, 'DBUSER');
my $dbpass   = metadataLookupStr($siteConfig, 'DBPASSWORD');
exit ($PS_EXIT_CONFIG_ERROR) unless defined $dbserver and $dbname and $dbuser and $dbpass;

my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";

my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n";

my $fileset_dir = "$dsroot/$product/$fileset";
my $index_script_name = "$fileset_dir/index.txt";

if ($del) {
    #
    # delete fileset
    #
    my $stmt = $dbh->prepare("SELECT prod_id, last_fs FROM dsProduct WHERE prod_name = \'$product\'");
    $stmt->execute();
    my $prod_row = $stmt->fetchrow_hashref();
    my $prod_id = $prod_row->{prod_id};
    my $old_last_fs = $prod_row->{last_fs};

    if (!$prod_id) {
        die("product $product not found\n");
    }
    $stmt = $dbh->prepare("SELECT fileset_id,fileset_name FROM dsFileset WHERE fileset_name = '$fileset'" .
                        " AND prod_id = $prod_id");

    $stmt->execute();
    my $fs_row = $stmt->fetchrow_hashref();

    if (!$fs_row) {
        print STDERR "Fileset '$fileset' not found under $product.\n";
        exit 1;
    }

    if ($remove) {
        if (system "rm -r $fileset_dir") {
            die("failed to remove $fileset_dir");
        }
    } else  {
        # zap the index script
        unlink("$index_script_name");
    }

    my $fileset_id = $fs_row->{fileset_id};

    $dbh->do("DELETE from dsFile where fileset_id = $fileset_id");
    $dbh->do("DELETE from dsFileset where fileset_id = $fileset_id");

    if ($old_last_fs eq $fs_row->{fileset_name}) {
        # print STDERR "deleting most recent fileset in $product\n";

        $stmt = $dbh->prepare("SELECT fileset_name from dsFileset " .
                    "WHERE prod_id = $prod_id ORDER BY reg_time DESC LIMIT 1");
        $stmt->execute();
        my $new_last_fs;
        my $new_newest = $stmt->fetchrow_hashref();
        if ($new_newest) {
            $new_last_fs = $new_newest->{fileset_name};
        } else {
            $new_last_fs = "none";
        }
        $dbh->do("UPDATE dsProduct SET last_fs = \'$new_last_fs\' WHERE prod_id = $prod_id");
    }
    exit 0;

} else {
    #
    # add a new fileset
    #

    if (!$datapath) {
        $datapath = $fileset_dir;
    }

    my $stmt = $dbh->prepare("SELECT prod_id FROM dsProduct WHERE prod_name = \'$product\'");
    $stmt->execute();
    my $row = $stmt->fetchrow_hashref();
    my $prod_id = $row->{prod_id};

    if (!$prod_id) {
        die("product $product not found");
    }
    my $count = $dbh->do("SELECT fileset_id FROM dsFileset WHERE fileset_name = '$fileset'" .
                        " AND prod_id = $prod_id");

    if ($count != 0E0) {
        print "Fileset '$fileset' already exists under $product.\n";
        exit 1;
    }

    # Note: There is a race condition here. Somebody could sneak in now and add a fileset with
    # fileset_name = $fileset. So later we'll check again that only one fileset with the name exists

    my $lineno = 0;

    my $in;
    if ($filelist eq "-") {
        $in = *STDIN;
    } else {
        open $in, "<$filelist" or die  "cannot open filelist file $filelist\n";
    }

    my @files;
    while (<$in>) {
        $lineno++;

        my $filename;
        my $filetype;
        my $ts_string = "";

        chomp;
        my @fields = split /\|/;
        if (@fields < 2) {
            die "malformed input line: $filelist line $lineno: $_\n";
        }
        $filename = shift @fields;
        $filetype = shift @fields;

        #
        # save any type specific fields provided in the string to be added to the
        #
        my $nfields = @fields;
        my $i;
        for ($i=0; $i < $nfields; $i++) {
            $ts_string .= ", \'$fields[$i]\'";
        }
        for ($i=$nfields ; $i < 4; $i++) {
            $ts_string .= ", NULL";
        }
        my $hashref = {
            'file'      => $filename,
            'type'      => $filetype,
            'ts_string' => $ts_string
        };

        push @files, $hashref;
    }

    die("empty filelist\n") if (@files == 0);

    # Prepare the fileset directory
    if ($linkfiles or $copyfiles) {
        ## create the fileset directory
        if (! -e $fileset_dir) {
            if (!mkdir $fileset_dir) {
                die("failed trying to create fileset directory $fileset_dir");
            }
        }
        eval {
            ## then copy or symlink the files into place
            foreach my $ref (@files) {
                my $src = "$datapath/$ref->{file}";
                my $dest = "$fileset_dir/$ref->{file}";

                if ($linkfiles) {
                    if (! symlink $src, $dest) {
                        die("failed trying to link $src to $dest");
                    }
                } else {
                    if (!copy($src, $dest)) {
                        die("copy($src, $dest) failed");
                    }
                }
            }
        };
        if ($@) { # an error occured
            print STDERR "error preparing the fileset directory: $@\n";

            if (!$no_cleanup && system "rm -r $fileset_dir") {
                die("failed to remove $fileset_dir");
            }

            exit $PS_EXIT_UNKNOWN_ERROR;
        }
    }

    # now calculate the md5sum and file size
    foreach my $file (@files) {

        my $path = "$fileset_dir/$file->{file}";
        my @finfo = stat("$path");

        unless (@finfo) {
            die ("Line $lineno: referenced file "
                ."($path does not exist.\n");
        }
        $file->{bytes}  = $finfo[7];

        # Get MD5 sum
        $file->{md5sum} = file_md5_hex($path);
    }


    # TODO: why do I do this before the database insert is complete?
    # create the cgi script index.txt by making a copy of its parent's
    if (!copy("$dsroot/$product/index.txt", $index_script_name)) {
        cleanup();
        die("failed to copy index script to file set");
    }

    eval {
        $dbh->{RaiseError} = 1; # raise exception if error occurs
        $dbh->{PrintError} = 1;
        $dbh->{AutoCommit} = 0;

        # note the resulting prodcolstr begins with a comma
        my $prodcolstr = make_prodcol_str($ps0, $ps1, $ps2, $ps3, $ps4, $ps5, $ps6, $ps7);

        my $qstring = "INSERT into dsFileset" .
                " (prod_id, fileset_name, reg_time, type," .
                " prod_col_0, prod_col_1, prod_col_2, prod_col_3, prod_col_4, prod_col_5, " .
                " prod_col_6, prod_col_7)" .
                " VALUES($prod_id, \'$fileset\', UTC_TIMESTAMP(), \'$fstype\' $prodcolstr)";

        $count = $dbh->do($qstring);
        if ($count == 0E0) {
            die("failed to insert $fileset");
        }

        my $fileset_id = $dbh->last_insert_id(undef, undef, undef, undef);

        # make sure that no-one got in and created a fileset with this name while we were busy
        $count = $dbh->do("SELECT fileset_id FROM dsFileset WHERE fileset_name = '$fileset'" .
                        " AND prod_id = $prod_id");
        if ($count > 1) {
            die("Fileset '$fileset' already exists under $product.");
        }

        foreach my $ref (@files) {
            # note: ts_string has a leading comma
            my $query = "INSERT INTO dsFile" .
                    " (fileset_id, file_id, file_name, bytes, md5sum, type, " .
                    " type_col_0, type_col_1, type_col_2, type_col_3)" .

                    " VALUES($fileset_id, 0, \'$ref->{file}\', $ref->{bytes}," .
                             " \'$ref->{md5sum}\', \'$ref->{type}\' $ref->{ts_string} )";

            $count = $dbh->do($query);

            if ($count == 0E0) {
                $dbh->rollback();
                die("failed to insert $ref->{file} into $fileset");
            }
        }

        $count = $dbh->do("UPDATE dsProduct SET last_update = UTC_TIMESTAMP(), last_fs = \'$fileset\'"
                                . " WHERE prod_id = $prod_id");
        if ($count == 0E0) {
            $dbh->rollback();
            die("failed to update dsProduct $prod_id");
        }

        $dbh->commit();
    };
    if ($@) { # an error occured
        print STDERR "transactionfailed, rolling back error was:\n$@\n";
        eval {$dbh->rollback();};
        cleanup();
        exit 1;
    }

    exit 0;
}


sub show_usage {
    my $str = shift;

    chomp $str;

    my @tmp = split(/\//, $0);

    pod2usage(
        -msg =>
"usage: $tmp[$#tmp] [--add|--del] --product prod_name --fileset fs_name --type fs_type --list filelist [options]

Commands:

    --add       Add a new fileset with the given metadata options.
                File information will be read per-line from the file specifed, in the form:

                    fileID|type|ts0|ts1|ts2|ts3

                These entries are read until EOF is sent.
                (The type specific ts[0-3]fields are optional)

    --del        Remove a fileset.

    --list      file containing list of files (use - for STDIN)

Options:
    --link      Link files from datapath to Data Store
    --copy      Copy files from datapath to Data Store
    --datapath  path to source files
    --ps0 - ps7 Optional product specific data
    --rm        with --del remove the fileset directory from the Data Store

$str",
        -exitval => 2
    );
}

sub make_prodcol_str {

    my @list = @_;
    my $string = "";

    foreach my $s (@list) {
        if ($s) {
            $string .= ", \'$s\'";
        } else {
            $string .= ", NULL";
        }
    }

    return $string;
}

sub cleanup {
    if (!$no_cleanup && ($linkfiles or $copyfiles)) {
        if (system "rm -r $fileset_dir") {
            print STDERR "failed to remove $fileset_dir";
        }
    } else {
        unlink($index_script_name);
    }
}
