Changeset 17883 for trunk/DataStoreServer/scripts/dsreg

Timestamp:

Jun 2, 2008, 3:03:40 PM (18 years ago)

Author:

bills

Message:

Lots of changes. Allow filelist to be specified on command line instead of
requiring from stdin. Sanity checking. Improvements in --link and
--copy operation

File:

• trunk/DataStoreServer/scripts/dsreg (modified) (26 diffs)

trunk/DataStoreServer/scripts/dsreg

@@ -1 +1 @@
 #!/usr/bin/env perl
 #
-# Data Store file set registration and de-registration
+# Data Store file set registration and de-registration program
 #
 
@@ -11 +11 @@
 use Digest::MD5::File qw( file_md5_hex );
 use DBI;
+#use DBD::mysql 4.005;  # needed to set this one day on ipp000
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::Stats;
@@ -29 +30 @@
 my $product;
 my $fileset;
+my $filelist;
 
 my $fstype;
-my $ps0;
-my $ps1;
-my $ps2;
-my $ps3;
-my $ps4;
-my $ps5;
-my $ps6;
-my $ps7;
+my ($ps0, $ps1, $ps2, $ps3, $ps4, $ps5, $ps6, $ps7);
 
 my $linkfiles;      # if set, put links to files in datapath in the Data Store
 my $copyfiles;      # if set, copy files from this directory to the Data Store fileset
 my $datapath;       # source for files required if $linkfiles or $copyfiles
+
 my $dbname;                     # Database name
-
-my $ds_dir;
+my $dsroot;
 
 my $add;
@@ -52 +47 @@
 
 my $verbosity = 1;
-
-
-
+my $no_cleanup = 1; # if something goes wrong don't delete copied or linked files for debug
 
 #
@@ -65 +58 @@
         'product=s'     =>      \$product,
         'fileset=s'     =>      \$fileset,
+        'list=s'        =>      \$filelist,
         'type=s'        =>      \$fstype,
         'datapath=s'    =>      \$datapath,
@@ -78 +72 @@
         'ps6=s'         =>      \$ps6,
         'ps7=s'         =>      \$ps7,
-           'dbname=s'   =>      \$dbname,
+        'dbname=s'      =>      \$dbname,
+        'dsroot=s'      =>      \$dsroot,
 ) or pod2usage(2);
 
 my $err = "";
 
-$err .= "product and fileset are required\n" unless defined $product and $fileset;
+$err .= "--product is required\n" unless defined $product;
+$err .= "--fileset is required\n" unless defined $fileset;
+
 
 # will exit if neither or both -add and -del were specified
@@ -90 +87 @@
 
 if ($add) {
-    if (!$fstype) {
-        $err .= "need to specify fileset type to add\n";
-    }
-
+    $err .= "--type is required\n" unless defined $fstype;
+    $err .= "--list is required\n" unless defined $filelist;
     if ($linkfiles and $copyfiles) {
-        $err .= "only one of --link and --copy allowedn";
+        $err .= "only one of --link and --copy is allowed\n";
     }
     if (($linkfiles or $copyfiles) and !$datapath) {
         $err .= "need to specify datapath with --link or --copy\n";
     }
+    if ($linkfiles && (substr($datapath, 0, 1) ne "/")) {
+        $err .= "datapath must begin with / when using --link\n";
+    }
 }
 
@@ -109 +107 @@
 my $ipprc =  PS::IPP::Config->new(); # IPP Configuration
 my $siteConfig = $ipprc->{_siteConfig};
-if (!$ds_dir) {
-    $ds_dir = metadataLookupStr($ipprc->{_siteConfig}, 'PSTAMP_DATA_STORE_ROOT');
-    if (!$ds_dir) {
+if (!$dsroot) {
+    $dsroot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');
+    if (!$dsroot) {
         die("Data Store root directory not set");
     }
@@ -117 +115 @@
 
 
-if (!stat("$ds_dir/index.txt")) {
-    $err .= "Data Store not found at '$ds_dir'.\n"
+if (!stat("$dsroot/index.txt")) {
+    $err .= "Data Store not found at '$dsroot'.\n"
 }
 
@@ -126 +124 @@
 
 show_usage("Invalid product '$product'.")
-    unless defined stat("$ds_dir/$product/index.txt");
+    unless defined stat("$dsroot/$product/index.txt");
 
 my $dbserver = metadataLookupStr($siteConfig, 'DBSERVER');
@@ -138 +136 @@
 my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n";
 
-
-
-my $fileset_dir = "$ds_dir/$product/$fileset";
+my $fileset_dir = "$dsroot/$product/$fileset";
 my $index_script_name = "$fileset_dir/index.txt";
 
@@ -154 +150 @@
 
     if (!$prod_id) {
-        die("product $product not found");
+        die("product $product not found\n");
     }
     $stmt = $dbh->prepare("SELECT fileset_id,fileset_name FROM dsFileset WHERE fileset_name = '$fileset'" .
@@ -226 +222 @@
     # fileset_name = $fileset. So later we'll check again that only one fileset with the name exists
 
-    # Read file data from STDIN
-    # XXX: Perhaps allow this to come from a file
-
     my $lineno = 0;
 
+    my $in;
+    if ($filelist eq "-") {
+        $in = *STDIN;
+    } else {
+        open $in, "<$filelist" or die  "cannot open filelist file $filelist\n";
+    }
+
     my @files;
-    while (<STDIN>) {
+    while (<$in>) {
         $lineno++;
 
@@ -242 +242 @@
         my @fields = split /\|/;
         if (@fields < 2) {
-                print STDERR "Line $lineno: ignored malformed input line.\n";
-                next;
+            die "malformed input line: $filelist line $lineno: $_\n";
         }
         $filename = shift @fields;
@@ -250 +249 @@
         #
         # save any type specific fields provided in the string to be added to the
-        # VALUES list for this file
+        #
         my $nfields = @fields;
         my $i;
@@ -259 +258 @@
             $ts_string .= ", NULL";
         }
-
-        my $path = "$datapath/$filename";
-        my @finfo = stat("$path");
-
-        unless (@finfo) {
-            die ("Line $lineno: referenced file "
-                ."($path does not exist.\n");
-        }
-
-        # Get MD5 sum
-        my $md5 = file_md5_hex($path);
-
         my $hashref = {
             'file'      => $filename,
-            'bytes'     => $finfo[7],
-            'md5sum'    => $md5,
             'type'      => $filetype,
-            'ts_string'  => $ts_string
+            'ts_string' => $ts_string
         };
 
@@ -282 +267 @@
     }
 
-    if (@files == 0) {
-        die("empty filelist");
-    }
-
-    #
+    die("empty filelist\n") if (@files == 0);
+
     # Prepare the fileset directory
     if ($linkfiles or $copyfiles) {
-            ## create the fileset directory
+        ## create the fileset directory
         if (! -e $fileset_dir) {
             if (!mkdir $fileset_dir) {
-                die("failed trying to make fileset directory $fileset_dir");
+                die("failed trying to create fileset directory $fileset_dir");
             }
         }
@@ -302 +284 @@
 
                 if ($linkfiles) {
-                    if (!  symlink $src, $dest) {
+                    if (! symlink $src, $dest) {
                         die("failed trying to link $src to $dest");
                     }
@@ -315 +297 @@
             print STDERR "error preparing the fileset directory: $@\n";
 
-            if (system "rm -r $fileset_dir") {
+            if (!$no_cleanup && system "rm -r $fileset_dir") {
                 die("failed to remove $fileset_dir");
             }
@@ -323 +305 @@
     }
 
+    # now calculate the md5sum and file size
+    foreach my $file (@files) {
+
+        my $path = "$fileset_dir/$file->{file}";
+        my @finfo = stat("$path");
+
+        unless (@finfo) {
+            die ("Line $lineno: referenced file "
+                ."($path does not exist.\n");
+        }
+        $file->{bytes}  = $finfo[7];
+
+        # Get MD5 sum
+        $file->{md5sum} = file_md5_hex($path);
+    }
+
+
+    # TODO: why do I do this before the database insert is complete?
     # create the cgi script index.txt by making a copy of its parent's
-    if (!copy("$ds_dir/$product/index.txt", $index_script_name)) {
+    if (!copy("$dsroot/$product/index.txt", $index_script_name)) {
         cleanup();
         die("failed to copy index script to file set");
@@ -331 +331 @@
     eval {
         $dbh->{RaiseError} = 1; # raise exception if error occurs
-        $dbh->{PrintError} = 0;
+        $dbh->{PrintError} = 1;
         $dbh->{AutoCommit} = 0;
 
@@ -360 +360 @@
             # note: ts_string has a leading comma
             my $query = "INSERT INTO dsFile" .
-
                     " (fileset_id, file_id, file_name, bytes, md5sum, type, " .
                     " type_col_0, type_col_1, type_col_2, type_col_3)" .
@@ -376 +375 @@
 
         $count = $dbh->do("UPDATE dsProduct SET last_update = UTC_TIMESTAMP(), last_fs = \'$fileset\'"
-            . " WHERE prod_id = $prod_id");
+                                . " WHERE prod_id = $prod_id");
         if ($count == 0E0) {
             $dbh->rollback();
@@ -404 +403 @@
     pod2usage(
         -msg =>
-"usage: $tmp[$#tmp] [--add|--del] --product prod_name --fileset fs_name --type fs_type [options]
+"usage: $tmp[$#tmp] [--add|--del] --product prod_name --fileset fs_name --type fs_type --list filelist [options]
 
 Commands:
 
-    --add        Add a new fileset with the given metadata options.
-                File information will be read per-line from STDIN, in the form:
+    --add       Add a new fileset with the given metadata options.
+                File information will be read per-line from the file specifed, in the form:
 
                     fileID|type|ts0|ts1|ts2|ts3
@@ -417 +416 @@
 
     --del        Remove a fileset.
+
+    --list      file containing list of files (use - for STDIN)
 
 Options:
@@ -447 +448 @@
 
 sub cleanup {
-
-    if ($linkfiles or $copyfiles) {
+    if (!$no_cleanup && ($linkfiles or $copyfiles)) {
         if (system "rm -r $fileset_dir") {
             print STDERR "failed to remove $fileset_dir";