#!/usr/bin/env perl
#
# Data Store file set registration and de-registration program
#

use strict;
use warnings;

use Getopt::Long qw( GetOptions );
use Pod::Usage qw( pod2usage );
use Digest::MD5::File qw( file_md5_hex );
use DBI;
use PS::IPP::Metadata::Config;
use PS::IPP::Metadata::Stats;
use PS::IPP::Metadata::List qw( parse_md_list );
use File::Copy;
use File::Basename;

use PS::IPP::Config qw($PS_EXIT_SUCCESS
                       $PS_EXIT_UNKNOWN_ERROR
                       $PS_EXIT_SYS_ERROR
                       $PS_EXIT_CONFIG_ERROR
                       $PS_EXIT_PROG_ERROR
                       $PS_EXIT_DATA_ERROR
                       $PS_EXIT_TIMEOUT_ERROR
                       metadataLookupStr
                       metadataLookupBool
                       caturi
                       );
my $product;
my $fileset;
my $filelist;

my $fstype;
my ($ps0, $ps1, $ps2, $ps3, $ps4, $ps5, $ps6, $ps7);

my $linkfiles;      # if set, put links to files in the Data Store. The actuall directory is $datapath
my $copyfiles;      # if set, copy files from this directory to the Data Store fileset
my $datapath;       # source for files required if $linkfiles or $copyfiles
my $abspath;			# Path specified is absolute

my $dbname;         # Database name
my $dsroot;         # root directory of the data store

my $add;
my $del;
my $remove;
my $force;          # on --del don't return error if fileset doesn't exist

my $verbose = 0;
my $no_cleanup = 0; # if something goes wrong don't delete copied or linked files for debug

# XXX allow group write permission: 
# TODO: set the group to a more restrictive one than users
umask 2;

#
# parse args
#

GetOptions(
        'add=s'         =>      \$add,
        'del=s'         =>      \$del,
        'product=s'     =>      \$product,
        'list=s'        =>      \$filelist,
        'type=s'        =>      \$fstype,
        'datapath=s'    =>      \$datapath,
	'abspath'       =>      \$abspath,
        'link'          =>      \$linkfiles,
        'copy'          =>      \$copyfiles,
        'rm'            =>      \$remove,
        'force'         =>      \$force,
        'ps0=s'         =>      \$ps0,          # product specific columns 0 - 7
        'ps1=s'         =>      \$ps1,
        'ps2=s'         =>      \$ps2,
        'ps3=s'         =>      \$ps3,
        'ps4=s'         =>      \$ps4,
        'ps5=s'         =>      \$ps5,
        'ps6=s'         =>      \$ps6,
        'ps7=s'         =>      \$ps7,
        'dbname=s'      =>      \$dbname,
        'dsroot=s'      =>      \$dsroot,
        'verbose'       =>      \$verbose,
) or pod2usage(2);

my $err = "";

$err .= "--product is required\n" unless defined $product;

# will exit if neither --add and --del is specified or if they both are specified
$err .= "Must specify either --add or --del.\n"
    unless $add xor $del;

if ($add) {
    $fileset = $add;
    $err .= "--type is required\n" unless defined $fstype;
    $err .= "--list is required\n" unless defined $filelist;
    if ($linkfiles and $copyfiles) {
        $err .= "only one of --link and --copy is allowed\n";
    }
    if (($linkfiles or $copyfiles) and !$datapath and !$abspath) {
        $err .= "need to specify datapath with --link or --copy\n";
    }
    if ($linkfiles && (substr($datapath, 0, 1) ne "/")) {
        $err .= "datapath must begin with / when using --link\n";
    }
} else {
    $fileset = $del;
}


#
# lookup the location of the Data Store
#

my $ipprc =  PS::IPP::Config->new(); # IPP Configuration
my $siteConfig = $ipprc->{_siteConfig};
if (!$dsroot) {
    $dsroot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');
    if (!$dsroot) {
        die("Data Store root directory not set");
    }
}


if (!stat("$dsroot/index.txt")) {
    $err .= "Data Store not found at '$dsroot'.\n"
}

# bail if any errors above
show_usage($err)
    if ($err);

show_usage("Invalid product '$product'.")
    unless defined stat("$dsroot/$product/index.txt");

$dbname      = metadataLookupStr($siteConfig, 'DBNAME') unless defined $dbname;
my $dbserver = metadataLookupStr($siteConfig, 'DBSERVER');
my $dbuser   = metadataLookupStr($siteConfig, 'DBUSER');
my $dbpass   = metadataLookupStr($siteConfig, 'DBPASSWORD');
exit ($PS_EXIT_CONFIG_ERROR) unless defined $dbserver and $dbname and $dbuser and $dbpass;

my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
my %conn_attrs = (PrintError => 1, RaiseError => 1, AutoCommit => 0);

my $dbh = DBI->connect_cached($dsn, $dbuser, $dbpass, \%conn_attrs)
        or die "Cannot connect to DB server\n";

print STDERR "dbh: $dbh\n" if $verbose;

my $fileset_dir = "$dsroot/$product/$fileset";
my $index_script_name = "$fileset_dir/index.txt";

if ($del) {
    #
    # delete fileset
    #

    my $stmt = $dbh->prepare("SELECT prod_id, last_fs FROM dsProduct WHERE prod_name = ?");
    $stmt->execute($product);

    my $prod_row = $stmt->fetchrow_hashref();
    my $prod_id = $prod_row->{prod_id};
    my $old_last_fs = $prod_row->{last_fs};

    if (!$prod_id) {
        die("product $product not found\n");
    }
    $stmt = $dbh->prepare("SELECT fileset_id,fileset_name FROM dsFileset" .
                                 " WHERE fileset_name = ? AND prod_id = $prod_id");

    $stmt->execute($fileset);
    my $fs_row = $stmt->fetchrow_hashref();

    if (!$fs_row) {
        print STDERR "Fileset '$fileset' not found in $product.\n";
        if ($force) {
            exit 0;
        } else {
            exit 1;
        }
    }
    $stmt->finish();

    my $fileset_id = $fs_row->{fileset_id};

    eval {
        $dbh->do("DELETE from dsFile where fileset_id = $fileset_id");
        $dbh->do("DELETE from dsFileset where fileset_id = $fileset_id");

        if ($old_last_fs eq $fs_row->{fileset_name}) {
            # print STDERR "deleting most recent fileset in $product\n";

            $stmt = $dbh->prepare("SELECT fileset_name from dsFileset " .
                        "WHERE prod_id = $prod_id ORDER BY reg_time DESC LIMIT 1");
            $stmt->execute();
            my $new_last_fs;
            my $new_newest = $stmt->fetchrow_hashref();
            if ($new_newest) {
                $new_last_fs = $new_newest->{fileset_name};
            } else {
                $new_last_fs = "none";
            }
            $stmt->finish();
            $dbh->do("UPDATE dsProduct SET last_fs = \'$new_last_fs\' WHERE prod_id = $prod_id");
        }
        $dbh->commit();
    };
    if ($@) { # an error occured
        print STDERR "transaction failed, rolling back error was:\n$@\n";
        eval {$dbh->rollback();};
        cleanup();
        exit 1;
    }

    $dbh->disconnect();

    if ($remove) {
        if (system "rm -r $fileset_dir") {
            die("failed to remove $fileset_dir");
        }
    } else  {
        # zap the index script
        unlink("$index_script_name");
    }
    exit 0;

} else {
    #
    # add a new fileset
    #

    if (!$datapath) {
        $datapath = $fileset_dir;
    }

    my $stmt = $dbh->prepare("SELECT prod_id FROM dsProduct WHERE prod_name = ?");
    $stmt->execute($product);
    my $row = $stmt->fetchrow_hashref();
    my $prod_id = $row->{prod_id};
    $stmt->finish();

    if (!$prod_id) {
        die("product $product not found");
    }
    my $count = $dbh->do("SELECT fileset_id FROM dsFileset WHERE fileset_name = ?" .
                        " AND prod_id = $prod_id", undef, ($fileset));
    if ($count != 0E0) {
        print STDERR "Fileset '$fileset' already exists in product $product.\n";
        exit 1;
    }

    # Note: There is a race condition here. Somebody could sneak in now and add a fileset with
    # fileset_name = $fileset. So later we'll check again that only one fileset with the name exists


    my $in;
    if ($filelist eq "-") {
        $in = *STDIN;
    } else {
        open $in, "<$filelist" or die  "cannot open filelist file $filelist\n";
    }

    my $lineno = 0;
    my @files;
    while (<$in>) {
        chomp;
        $lineno++;

        my $filename;
        my $bytes;
        my $md5sum;
        my $filetype;

        my @fields = split /\|/;
        if (@fields < 4) {
            die "malformed input line at $filelist line $lineno: $_\n";
        }
        $filename = shift @fields;
        $bytes    = shift @fields;  # if this is null it will be calculated below
        $md5sum   = shift @fields;  # if this is null it will be calculated below
        $filetype = shift @fields;

        # make sure the length of the type specific columns is 8
        while (@fields < 8) {
            push @fields, undef;
        }
        my $ts_cols = [@fields];

        my $hashref = {
            'file'      => $filename,
            'bytes'     => $bytes,
            'md5sum'    => $md5sum,
            'type'      => $filetype,
            'ts_cols'   => $ts_cols,
        };

        push @files, $hashref;
    }

    die("empty filelist\n") if (@files == 0);

    # Prepare the fileset directory
    if ($linkfiles or $copyfiles) {
        ## create the fileset directory
        if (! -e $fileset_dir) {
            if (!mkdir $fileset_dir) {
                die("failed trying to create fileset directory $fileset_dir");
            }
        }
        eval {
            ## then copy or symlink the files into place
            foreach my $ref (@files) {
                my $src = (defined $abspath ? '' : "$datapath/") . "$ref->{file}";
		my $filename = fileparse($ref->{file}, ());
                my $dest = "$fileset_dir/$filename";

                if ($linkfiles) {
                    if (! symlink $src, $dest) {
                        die("failed trying to link $src to $dest");
                    }
                } else {
                    if (!copy($src, $dest)) {
                        die("copy($src, $dest) failed");
                    }
                }
            }
        };
        if ($@) { # an error occured
            print STDERR "error preparing the fileset directory: $@\n";

            if (!$no_cleanup && system "rm -r $fileset_dir") {
                die("failed to remove $fileset_dir");
            }
            exit $PS_EXIT_UNKNOWN_ERROR;
        }
    }

    # now calculate the md5sum and file size if they weren't provided
    foreach my $file (@files) {
	my $filename = fileparse($file->{file}, ());
        my $path = "$fileset_dir/$filename";
        if (! -e $path ) {
            die "file $path not found";
        }
        if (!$file->{bytes}) {
            my @finfo = stat($path);
            unless (@finfo) {
                die ("can't stat $path");
            }
            $file->{bytes}  = $finfo[7];
        }
        if (!$file->{md5sum}) {
            # Get MD5 sum
            $file->{md5sum} = file_md5_hex($path);
        }
    }


    if (! $dbh->ping()) {
        # database connection has gone away, reconnect
        $dbh->disconnect();
        $dbh = DBI->connect_cached($dsn, $dbuser, $dbpass, \%conn_attrs)
            or die "Cannot connect to server\n";
        print STDERR "ping failed new dbh: $dbh\n" if $verbose;
    }

    eval {
        my $qstring = "INSERT into dsFileset" .
                " (prod_id, fileset_name, reg_time, type," .
                " prod_col_0, prod_col_1, prod_col_2, prod_col_3, prod_col_4, prod_col_5, " .
                " prod_col_6, prod_col_7)" .
                " VALUES(?, ?, UTC_TIMESTAMP(), ?, ?, ?, ?, ?, ?, ?, ?, ?)";

        $count = $dbh->do($qstring, undef, ($prod_id, $fileset, $fstype, 
                            $ps0, $ps1, $ps2, $ps3, $ps4, $ps5, $ps6, $ps7));

        die("failed to insert $fileset") if ($count == 0E0);

        my $fileset_id = $dbh->last_insert_id(undef, undef, undef, undef);

        # make sure that no-one got in and created a fileset with this name while we were busy
        $count = $dbh->do("SELECT fileset_id FROM dsFileset WHERE fileset_name = ?" .
                        " AND prod_id = $prod_id", undef, ($fileset));
        if ($count > 1) {
            die("Fileset '$fileset' already exists under $product.");
        }

        foreach my $ref (@files) {
	    my $filename = fileparse($ref->{file}, ());
            my $query = "INSERT INTO dsFile" .  
                    " (fileset_id, file_id, file_name, bytes, md5sum, type," .
                    " type_col_0, type_col_1, type_col_2, type_col_3, type_col_4, type_col_5," .
                    " type_col_6, type_col_7)" .
                    " VALUES($fileset_id, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";

            my $aref = $ref->{ts_cols};
            my $do_args =  [($filename, $ref->{bytes}, $ref->{md5sum}, $ref->{type}), @$aref];

            $count = $dbh->do($query, undef, @$do_args);

            if ($count == 0E0) {
                die("failed to insert $filename into $fileset");
            }
        }

        $count = $dbh->do("UPDATE dsProduct SET last_update = UTC_TIMESTAMP(), last_fs = \'$fileset\'"
                                . " WHERE prod_id = $prod_id");
        if ($count == 0E0) {
            die("failed to update dsProduct $prod_id");
        }

        $dbh->commit();

        # create the cgi script index.txt by making a copy of its parent's
        if (!copy("$dsroot/$product/index.txt", $index_script_name)) {
            die("failed to copy index script to file set");
        }
    };
    if ($@) { # an error occured
        print STDERR "transaction failed, rolling back error was:\n$@\n";
        eval {$dbh->rollback();};
        cleanup();
        exit 1;
    }

    print STDERR "calling disconnect before exit\n" if $verbose;
    $dbh->disconnect();

    exit 0;
}


sub show_usage {
    my $str = shift;

    chomp $str;

    my @tmp = split(/\//, $0);

    pod2usage(
        -msg =>
"usage: $tmp[$#tmp] --add fs_name |--del fs_name --product prod_name --type fs_type --list filelist [options]

Commands:

    --add       Add a new fileset with the given metadata to the specified product.
                File information will be read per-line from the file specifed by the
                list option.
                Input lines are in the following format:

                    fileID|size|md5sum|type|ts0|ts1|ts2|ts3

                These entries are read until EOF is reached.

                If either size or md5sum are left blank they will be calculated by dsreg

                (The type specific fields ts[0-3]fields are optional and may be omitted)

    --del       Remove the fileset fs_name.



Options:
    --product   the product for the given file set (required)
    --list      file containing list of files (use - for STDIN) (required)
    --link      Link files from datapath to Data Store
    --copy      Copy files from datapath to Data Store
    --datapath  path to source files for --copy or --link
    --ps0 - ps7 Optional product specific data
    --rm        with --del remove the fileset directory from the Data Store
    --force     with --del do not return error status if the fileset does not exist
    --dbname    select the Data Store's database name
                (usually required depending on configuration)

$str",
        -exitval => 2
    );
}

sub cleanup {
    if (!$no_cleanup && ($linkfiles or $copyfiles)) {
        if (system "rm -r $fileset_dir") {
            print STDERR "failed to remove $fileset_dir";
        }
    } else {
        unlink($index_script_name);
    }
}
