Index: trunk/DataStoreServer/scripts/dsreg =================================================================== --- trunk/DataStoreServer/scripts/dsreg (revision 16765) +++ trunk/DataStoreServer/scripts/dsreg (revision 16867) @@ -1,7 +1,5 @@ #!/usr/bin/env perl # -# DataStore fileset registration -# -# Does not enforce validity of metadata items. +# Data Store file set registration and de-registration # @@ -29,23 +27,31 @@ caturi ); +my $product; +my $fileset; + +my $fstype; +my $ps0; +my $ps1; +my $ps2; +my $ps3; +my $ps4; +my $ps5; +my $ps6; +my $ps7; + +my $linkfiles; # if set, put links to files in datapath in the Data Store +my $copyfiles; # if set, copy files from this directory to the Data Store fileset +my $datapath; # source for files required if $linkfiles or $copyfiles + +my $ds_dir; my $add; my $del; - -my $ds_dir; -my $product; -my $fileset; - -### XXX: these aren't currently used -### fill in some bogus values here -my $type = 'OBJECT'; -my $ra = '00:00:00.00'; -my $dec = '00:00:00.00'; -my $equinox = 2000; -my $etime = 30.0; -my $filter = 'z'; -my $airmass = 1.2; - -my $date; +my $remove; + +my $verbosity = 1; + + + # @@ -53,27 +59,49 @@ # -# XXX: need to make argument parsing file type specific GetOptions( - 'add' => \$add, - 'del' => \$del, - 'dsdir|ds=s' => \$ds_dir, - 'type|t=s' => \$type, - 'ra|r=s' => \$ra, - 'dec|d=s' => \$dec, - 'equinox|eq=f' => \$equinox, - 'etime|ex|e=f' => \$etime, - 'filter|f=s' => \$filter, - 'date=s' => \$date, - 'airmass|a=f' => \$airmass, + 'add' => \$add, + 'del' => \$del, + 'product=s' => \$product, + 'fileset=s' => \$fileset, + 'type=s' => \$fstype, + 'datapath=s' => \$datapath, + 'link' => \$linkfiles, + 'copy' => \$copyfiles, + 'rm' => \$remove, + 'ps0=s' => \$ps0, # product specific columns 0 - 7 + 'ps1=s' => \$ps1, + 'ps2=s' => \$ps2, + 'ps3=s' => \$ps3, + 'ps4=s' => \$ps4, + 'ps5=s' => \$ps5, + 'ps6=s' => \$ps6, + 'ps7=s' => \$ps7 ) or pod2usage(2); my $err = ""; -$err .= "Must specify a product and a fileset.\n" - unless @ARGV == 2; +$err .= "product and fileset are required\n" unless defined $product and $fileset; # will exit if neither or both -add and -del were specified $err .= "Must specify either --add or --del.\n" unless $add xor $del; + +if ($add) { + if (!$fstype) { + $err .= "need to specify fileset type to add\n"; + } + + if ($linkfiles and $copyfiles) { + $err .= "only one of --link and --copy allowedn"; + } + if (($linkfiles or $copyfiles) and !$datapath) { + $err .= "need to specify datapath with --link or --copy\n"; + } +} + + +# +# lookup the location of the Data Store +# my $ipprc = PS::IPP::Config->new(); # IPP Configuration @@ -82,5 +110,5 @@ $ds_dir = metadataLookupStr($ipprc->{_siteConfig}, 'PSTAMP_DATA_STORE_ROOT'); if (!$ds_dir) { - die("data store root not defined"); + die("Data Store root directory not set"); } } @@ -88,5 +116,5 @@ if (!stat("$ds_dir/index.txt")) { - $err .= "Datastore not found at '$ds_dir'.\n" + $err .= "Data Store not found at '$ds_dir'.\n" } @@ -95,15 +123,6 @@ if ($err); -$product = shift; - show_usage("Invalid product '$product'.") unless defined stat("$ds_dir/$product/index.txt"); - -$fileset = shift; - - -# -# run -# my $dbserver = metadataLookupStr($siteConfig, 'DBSERVER'); @@ -117,5 +136,8 @@ my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n"; -my $index_script_name = "$ds_dir/$product/$fileset/index.txt"; + + +my $fileset_dir = "$ds_dir/$product/$fileset"; +my $index_script_name = "$fileset_dir/index.txt"; if ($del) { @@ -143,6 +165,12 @@ } - # zap the index script - unlink("$index_script_name"); + if ($remove) { + if (system "rm -r $fileset_dir") { + die("failed to remove $fileset_dir"); + } + } else { + # zap the index script + unlink("$index_script_name"); + } my $fileset_id = $fs_row->{fileset_id}; @@ -170,6 +198,10 @@ } else { # - # adding a new fileset + # add a new fileset # + + # make a string out of the product specifc column values provided + my $prodcolstr = make_prodcol_str($ps0, $ps1, $ps2, $ps3, $ps4, $ps5, $ps6, $ps7); + my $stmt = $dbh->prepare("SELECT prod_id FROM dsProduct WHERE prod_name = \'$product\'"); $stmt->execute(); @@ -198,47 +230,93 @@ my $filename; my $filetype; - my $type_col_0; - if (/([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)/) { - $filename = $1; - $filetype = $2; - $type_col_0 = $3; - } else { - unless (/([A-Za-z0-9-_.]+)\s+([A-Za-z0-9-_.]+)\s+/) { + my $ts_string = ""; + + chomp; + my @fields = split /\|/; + if (@fields < 2) { print STDERR "Line $lineno: ignored malformed input line.\n"; next; - } - $filename = $1; - $filetype = $2; - } - - my @finfo = stat("$ds_dir/$product/$fileset/$filename"); + } + $filename = shift @fields; + $filetype = shift @fields; + + # + # save any type specific fields provided in the string to be added to the + # VALUES list for this file + my $nfields = @fields; + my $i; + for ($i=0; $i < $nfields; $i++) { + $ts_string .= ", \'$fields[$i]\'"; + } + for ($i=$nfields ; $i < 4; $i++) { + $ts_string .= ", NULL"; + } + + my $path = "$datapath/$filename"; + my @finfo = stat("$path"); unless (@finfo) { die ("Line $lineno: referenced file " - ."($ds_dir/$product/$fileset/$1) does not exist.\n"); + ."($path does not exist.\n"); } # Get MD5 sum - my $md5 = file_md5_hex("$ds_dir/$product/$fileset/$filename"); + my $md5 = file_md5_hex($path); my $hashref = { - 'file' => $filename, - 'bytes' => $finfo[7], + 'file' => $filename, + 'bytes' => $finfo[7], 'md5sum' => $md5, 'type' => $filetype, - 'type_col_0'=> $type_col_0 + 'ts_string' => $ts_string }; push @files, $hashref; } - if (@files == 0) { die("empty filelist"); } - - # create the cgi script index file by making a copy of its parent's + + # + # Prepare the fileset directory + if ($linkfiles or $copyfiles) { + ## create the fileset directory + if (! -e $fileset_dir) { + if (!mkdir $fileset_dir) { + die("failed trying to make fileset directory $fileset_dir"); + } + } + eval { + ## then copy or symlink the files into place + foreach my $ref (@files) { + my $src = "$datapath/$ref->{file}"; + my $dest = "$fileset_dir/$ref->{file}"; + + if ($linkfiles) { + if (! symlink $src, $dest) { + die("failed trying to link $src to $dest"); + } + } else { + if (!copy($src, $dest)) { + die("copy($src, $dest) failed"); + } + } + } + }; + if ($@) { # an error occured + print STDERR "error preparing the fileset directory: $@\n"; + + if (system "rm -r $fileset_dir") { + die("failed to remove $fileset_dir"); + } + + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + + # create the cgi script index.txt by making a copy of its parent's if (!copy("$ds_dir/$product/index.txt", $index_script_name)) { - die("failed to copy index script to fileset"); + die("failed to copy index script to file set"); } @@ -248,7 +326,11 @@ $dbh->{AutoCommit} = 0; - $count = $dbh->do("INSERT into dsFileset" . - " (prod_id, fileset_name, reg_time, type)" . - " VALUES($prod_id, \'$fileset\', UTC_TIMESTAMP(), 'PSRESULTS')"); + my $qstring = "INSERT into dsFileset" . + " (prod_id, fileset_name, reg_time, type," . + " prod_col_0, prod_col_1, prod_col_2, prod_col_3, prod_col_4, prod_col_5, " . + " prod_col_6, prod_col_7)" . + # note prodcolstr begins with a comma + " VALUES($prod_id, \'$fileset\', UTC_TIMESTAMP(), \'$fstype\' $prodcolstr)"; + $count = $dbh->do($qstring); if ($count == 0E0) { die("failed to insert $fileset"); @@ -260,16 +342,13 @@ foreach my $ref (@files) { - my $query; - if ($ref->{type_col_0}) { - $query = "INSERT INTO dsFile" . - " (fileset_id, file_id, file_name, bytes, md5sum, type, type_col_0)" . + # note: ts_string has a leading comma + my $query = "INSERT INTO dsFile" . + + " (fileset_id, file_id, file_name, bytes, md5sum, type, " . + " type_col_0, type_col_1, type_col_2, type_col_3)" . + " VALUES($fileset_id, 0, \'$ref->{file}\', $ref->{bytes}," . - " \'$ref->{md5sum}\', \'$ref->{type}\', \'$ref->{type_col_0}\')"; - } else { - $query = "INSERT INTO dsFile" . - " (fileset_id, file_id, file_name, bytes, md5sum, type)" . - " VALUES($fileset_id, 0, \'$ref->{file}\', $ref->{bytes}," . - " \'$ref->{md5sum}\', \'$ref->{type}\')"; - } + " \'$ref->{md5sum}\', \'$ref->{type}\' $ref->{ts_string} )"; + $count = $dbh->do($query); @@ -301,4 +380,5 @@ } + sub show_usage { my $str = shift; @@ -310,42 +390,43 @@ pod2usage( -msg => -"usage: $tmp[$#tmp] [--add|--del] product fileset [options] +"usage: $tmp[$#tmp] [--add|--del] --product prod_name --fileset fs_name --type fs_type [options] Commands: - --del Remove a fileset. --add Add a new fileset with the given metadata options. File information will be read per-line from STDIN, in the form: - fileID type [chipname] + + fileID|type|ts0|ts1|ts2|ts3 + These entries are read until EOF is sent. + (The type specific ts[0-3]fields are optional) + + --del Remove a fileset. + Options: - - --dsdir Specify the path to the Datastore + --link Link files from datapath to Data Store + --copy Copy files from datapath to Data Store + --datapath path to source files + --ps0 - ps7 Optional product specific data + --rm with --del remove the fileset directory from the Data Store $str", -exitval => 2 ); - - if (0) { - my $saveString = "(The following metadata is not used yet) - - --date ISO8601 date string. (File timestamp used if not specified.) - --type Fileset type. - --ra Telescope pointing RA. - --dec Telescope pointing Dec. - --equinox Equinox. - --etime Exposure time. - --filter Filter used. - --airmass Air mass. "; - } -} - -# get the current UTC time in iso8601 format -# this is used if no time parameter is specified -sub get_iso8601() { - my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = - gmtime(time); - - return sprintf("%4d-%02d-%02dT%02d:%02d:%02dZ", - $year+1900, $mon+1, $mday, $hour, $min, $sec); -} +} + +sub make_prodcol_str { + + my @list = @_; + my $string = ""; + + foreach my $s (@list) { + if ($s) { + $string .= ", \'$s\'"; + } else { + $string .= ", NULL"; + } + } + + return $string; +}