#!/usr/bin/env perl
#
# Data Store file set registration and de-registration
#

use strict;
use warnings;

use Getopt::Long qw( GetOptions );
use Pod::Usage qw( pod2usage );
use Digest::MD5::File qw( file_md5_hex ); 
use DBI;
use PS::IPP::Metadata::Config;
use PS::IPP::Metadata::Stats;
use PS::IPP::Metadata::List qw( parse_md_list );
use File::Copy;

use PS::IPP::Config qw($PS_EXIT_SUCCESS
		       $PS_EXIT_UNKNOWN_ERROR
		       $PS_EXIT_SYS_ERROR
		       $PS_EXIT_CONFIG_ERROR
		       $PS_EXIT_PROG_ERROR
		       $PS_EXIT_DATA_ERROR
		       $PS_EXIT_TIMEOUT_ERROR
		       metadataLookupStr
		       metadataLookupBool
		       caturi
		       );
my $product;
my $fileset;

my $fstype;
my $ps0;
my $ps1;
my $ps2;
my $ps3;
my $ps4;
my $ps5;
my $ps6;
my $ps7;

my $linkfiles;      # if set, put links to files in datapath in the Data Store
my $copyfiles;      # if set, copy files from this directory to the Data Store fileset
my $datapath;       # source for files required if $linkfiles or $copyfiles

my $ds_dir;

my $add;
my $del;
my $remove;

my $verbosity = 1;




#
# parse args
#

GetOptions(
        'add'           =>      \$add,
        'del'           =>      \$del,
	'product=s'	=>	\$product,
	'fileset=s'	=>	\$fileset,
	'type=s'	=>	\$fstype,
        'datapath=s'    =>      \$datapath,
        'link'          =>      \$linkfiles,
        'copy'          =>      \$copyfiles,
        'rm'            =>      \$remove,
	'ps0=s'		=>	\$ps0,          # product specific columns 0 - 7
	'ps1=s'		=>	\$ps1,
	'ps2=s'		=>	\$ps2,
	'ps3=s'		=>	\$ps3,
	'ps4=s'		=>	\$ps4,
	'ps5=s'		=>	\$ps5,
	'ps6=s'		=>	\$ps6,
	'ps7=s'		=>	\$ps7
) or pod2usage(2);

my $err = "";

$err .= "product and fileset are required\n" unless defined $product and $fileset;

# will exit if neither or both -add and -del were specified
$err .= "Must specify either --add or --del.\n"
    unless $add xor $del;

if ($add) {
    if (!$fstype) {
        $err .= "need to specify fileset type to add\n";
    }

    if ($linkfiles and $copyfiles) {
        $err .= "only one of --link and --copy allowedn";
    }
    if (($linkfiles or $copyfiles) and !$datapath) {
        $err .= "need to specify datapath with --link or --copy\n";
    }
}


#
# lookup the location of the Data Store
#

my $ipprc =  PS::IPP::Config->new(); # IPP Configuration
my $siteConfig = $ipprc->{_siteConfig};
if (!$ds_dir) {
    $ds_dir = metadataLookupStr($ipprc->{_siteConfig}, 'PSTAMP_DATA_STORE_ROOT');
    if (!$ds_dir) {
        die("Data Store root directory not set");
    }
}


if (!stat("$ds_dir/index.txt")) {
    $err .= "Data Store not found at '$ds_dir'.\n"
}

# bail if any of the above args were improper
show_usage($err)
    if ($err);

show_usage("Invalid product '$product'.")
    unless defined stat("$ds_dir/$product/index.txt");

my $dbserver = metadataLookupStr($siteConfig, 'DBSERVER');
my $dbname   = metadataLookupStr($siteConfig, 'DBNAME');
my $dbuser   = metadataLookupStr($siteConfig, 'DBUSER');
my $dbpass   = metadataLookupStr($siteConfig, 'DBPASSWORD');
exit ($PS_EXIT_CONFIG_ERROR) unless defined $dbserver and $dbname and $dbuser and $dbpass;

my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";

my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n";



my $fileset_dir = "$ds_dir/$product/$fileset";
my $index_script_name = "$fileset_dir/index.txt";

if ($del) {
    #
    # delete fileset
    #
    my $stmt = $dbh->prepare("SELECT prod_id, last_fs FROM dsProduct WHERE prod_name = \'$product\'");
    $stmt->execute();
    my $prod_row = $stmt->fetchrow_hashref();
    my $prod_id = $prod_row->{prod_id};
    my $old_last_fs = $prod_row->{last_fs};

    if (!$prod_id) {
        die("product $product not found");
    }
    $stmt = $dbh->prepare("SELECT fileset_id,fileset_name FROM dsFileset WHERE fileset_name = '$fileset'" .
                        " AND prod_id = $prod_id");

    $stmt->execute();
    my $fs_row = $stmt->fetchrow_hashref();

    if (!$fs_row) {
        print "Fileset '$fileset' not found under $product.\n";
        exit 1;
    }

    if ($remove) {
        if (system "rm -r $fileset_dir") {
            die("failed to remove $fileset_dir");
        }
    } else  {
        # zap the index script
        unlink("$index_script_name");
    }

    my $fileset_id = $fs_row->{fileset_id};

    $dbh->do("DELETE from dsFile where fileset_id = $fileset_id");
    $dbh->do("DELETE from dsFileset where fileset_id = $fileset_id");

    if ($old_last_fs eq $fs_row->{fileset_name}) {
        # print STDERR "deleting most recent fileset in $product\n";

        $stmt = $dbh->prepare("SELECT fileset_name from dsFileset " .
                    "WHERE prod_id = $prod_id ORDER BY reg_time DESC LIMIT 1");
        $stmt->execute();
        my $new_last_fs;
        my $new_newest = $stmt->fetchrow_hashref();
        if ($new_newest) {
            $new_last_fs = $new_newest->{fileset_name};
        } else {
            $new_last_fs = "none";
        }
        $dbh->do("UPDATE dsProduct SET last_fs = \'$new_last_fs\' WHERE prod_id = $prod_id");
    }
    exit 0;

} else {
    #
    # add a new fileset
    #

    # make a string out of the product specifc column values provided
    my $prodcolstr = make_prodcol_str($ps0, $ps1, $ps2, $ps3, $ps4, $ps5, $ps6, $ps7);

    my $stmt = $dbh->prepare("SELECT prod_id FROM dsProduct WHERE prod_name = \'$product\'");
    $stmt->execute();
    my $row = $stmt->fetchrow_hashref();
    my $prod_id = $row->{prod_id};

    if (!$prod_id) {
        die("product $product not found");
    }
    my $count = $dbh->do("SELECT fileset_id FROM dsFileset WHERE fileset_name = '$fileset'" .
                        " AND prod_id = $prod_id");

    if ($count != 0E0) {
        print "Fileset '$fileset' already exists under $product.\n";
        exit 1;
    }

    # Read file data from STDIN

    my $lineno = 0;

    my @files;
    while (<STDIN>) {
        $lineno++;
        
        my $filename;
        my $filetype;
        my $ts_string = "";

        chomp;
        my @fields = split /\|/;
        if (@fields < 2) {
                print STDERR "Line $lineno: ignored malformed input line.\n";
                next;
        }
        $filename = shift @fields;
        $filetype = shift @fields;

        # 
        # save any type specific fields provided in the string to be added to the
        # VALUES list for this file
        my $nfields = @fields;
        my $i;
        for ($i=0; $i < $nfields; $i++) {
            $ts_string .= ", \'$fields[$i]\'";
        }
        for ($i=$nfields ; $i < 4; $i++) {
            $ts_string .= ", NULL";
        }

        my $path = "$datapath/$filename";
        my @finfo = stat("$path");

        unless (@finfo) {
            die ("Line $lineno: referenced file "
                ."($path does not exist.\n");
        }
        
        # Get MD5 sum
        my $md5 = file_md5_hex($path);

        my $hashref = { 
            'file'      => $filename,
            'bytes'     => $finfo[7],
            'md5sum'    => $md5,
            'type'      => $filetype,
            'ts_string'  => $ts_string
        };

        push @files, $hashref;
    }

    if (@files == 0) {
        die("empty filelist");
    }
    
    #
    # Prepare the fileset directory
    if ($linkfiles or $copyfiles) {
            ## create the fileset directory
        if (! -e $fileset_dir) {
            if (!mkdir $fileset_dir) {
                die("failed trying to make fileset directory $fileset_dir");
            }
        }
        eval {
            ## then copy or symlink the files into place
            foreach my $ref (@files) {
                my $src = "$datapath/$ref->{file}";
                my $dest = "$fileset_dir/$ref->{file}";

                if ($linkfiles) {
                    if (!  symlink $src, $dest) {
                        die("failed trying to link $src to $dest");
                    }
                } else {
                    if (!copy($src, $dest)) {
                        die("copy($src, $dest) failed");
                    }
                }
            }
        };
        if ($@) { # an error occured
            print STDERR "error preparing the fileset directory: $@\n";

            if (system "rm -r $fileset_dir") {
                die("failed to remove $fileset_dir");
            }
            
            exit $PS_EXIT_UNKNOWN_ERROR;
        }
    }

    # create the cgi script index.txt by making a copy of its parent's
    if (!copy("$ds_dir/$product/index.txt", $index_script_name)) {
        die("failed to copy index script to file set");
    }

    eval {
        $dbh->{RaiseError} = 1; # raise exception if error occurs
        $dbh->{PrintError} = 0;
        $dbh->{AutoCommit} = 0;

        my $qstring = "INSERT into dsFileset" .
                " (prod_id, fileset_name, reg_time, type," .
                " prod_col_0, prod_col_1, prod_col_2, prod_col_3, prod_col_4, prod_col_5, " .
                " prod_col_6, prod_col_7)" .
                # note prodcolstr begins with a comma
                " VALUES($prod_id, \'$fileset\', UTC_TIMESTAMP(), \'$fstype\' $prodcolstr)";
        $count = $dbh->do($qstring);
        if ($count == 0E0) {
            die("failed to insert $fileset");
        }

        my $fileset_id = $dbh->last_insert_id(undef, undef, undef, undef);

        # print STDERR "Inserted fileset_id $fileset_id\n";

        foreach my $ref (@files) {
            # note: ts_string has a leading comma
            my $query = "INSERT INTO dsFile" .

                    " (fileset_id, file_id, file_name, bytes, md5sum, type, " .
                    " type_col_0, type_col_1, type_col_2, type_col_3)" .

                    " VALUES($fileset_id, 0, \'$ref->{file}\', $ref->{bytes}," .
                             " \'$ref->{md5sum}\', \'$ref->{type}\' $ref->{ts_string} )";

            $count = $dbh->do($query);

            if ($count == 0E0) {
                $dbh->rollback();
                die("failed to insert $ref->{file} into $fileset");
            }
        }

        $count = $dbh->do("UPDATE dsProduct SET last_update = UTC_TIMESTAMP(), last_fs = \'$fileset\'"
            . " WHERE prod_id = $prod_id");
        if ($count == 0E0) {
            $dbh->rollback();
            die("failed to update dsProduct $prod_id");
        }

        $dbh->commit();
    };
    if ($@) { # an error occured
        print STDERR "transactionfailed, rolling back error was:\n$@\n";
        eval {$dbh->rollback();};
        unlink($index_script_name);
        exit 1;
    }

    # print "Added $fileset to $product.\n";

    exit 0;
}


sub show_usage {
    my $str = shift;

    chomp $str;

    my @tmp = split(/\//, $0);

    pod2usage(
        -msg => 
"usage: $tmp[$#tmp] [--add|--del] --product prod_name --fileset fs_name --type fs_type [options]

Commands:

    --add        Add a new fileset with the given metadata options.
                File information will be read per-line from STDIN, in the form:

                    fileID|type|ts0|ts1|ts2|ts3

                These entries are read until EOF is sent.
                (The type specific ts[0-3]fields are optional)

    --del        Remove a fileset.

Options:
    --link      Link files from datapath to Data Store
    --copy      Copy files from datapath to Data Store
    --datapath  path to source files
    --ps0 - ps7 Optional product specific data
    --rm        with --del remove the fileset directory from the Data Store
    
$str",
        -exitval => 2
    );
}

sub make_prodcol_str {

    my @list = @_;
    my $string = "";

    foreach my $s (@list) {
        if ($s) {
            $string .= ", \'$s\'";
        } else {
            $string .= ", NULL";
        }
    }

    return $string;
}
