Index: trunk/Nebulous-Server/Build.PL =================================================================== --- trunk/Nebulous-Server/Build.PL (revision 25010) +++ trunk/Nebulous-Server/Build.PL (revision 26292) @@ -46,4 +46,5 @@ bin/neb-fsck bin/neb-initdb + bin/neb-cabadd bin/neb-voladd bin/neb-voladm Index: trunk/Nebulous-Server/MANIFEST =================================================================== --- trunk/Nebulous-Server/MANIFEST (revision 25010) +++ trunk/Nebulous-Server/MANIFEST (revision 26292) @@ -8,4 +8,6 @@ bin/neb-initdb bin/neb-voladd +bin/neb-cabadd +bin/neb-voladm bin/nebdiskd docs/database_setup.txt Index: trunk/Nebulous-Server/bin/neb-cabadd =================================================================== --- trunk/Nebulous-Server/bin/neb-cabadd (revision 26292) +++ trunk/Nebulous-Server/bin/neb-cabadd (revision 26292) @@ -0,0 +1,178 @@ +#!/usr/bin/env perl + +# Copyright (C) 2005-2008 Joshua Hoblitt +# +# $Id: neb-cabadd,v 1.2 2008-10-01 21:05:57 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.02'; + +use DBI; +use Nebulous::Server::SQL; +use URI::file; +use URI; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); + +my ($db, $dbhost, $dbuser, $dbpass, $cname, $location); + +$db = $ENV{'NEB_DB'} unless $db; +$dbhost = $ENV{'NEB_DBHOST'} || 'localhost'; +$dbuser = $ENV{'NEB_USER'} unless $dbuser; +$dbpass = $ENV{'NEB_PASS'} unless $dbpass; + +GetOptions( + 'db|d=s' => \$db, + 'host=s' => \$dbhost, + 'pass|p=s' => \$dbpass, + 'user|u=s' => \$dbuser, + 'cname|n=s' => \$cname, + 'location|l=s' => \$location, +) || pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --db --user --pass --cname", -exitval => 2 ) + unless $db && $dbuser && $dbpass && $cname; + +my $dbh = DBI->connect( + "DBI:mysql:database=$db:host=$dbhost", + $dbuser, + $dbpass, + { + RaiseError => 1, + PrintError => 0, + AutoCommit => 1, + }, +); + +my $sql = Nebulous::Server::SQL->new(); + +print "Adding cabinet..."; + +my $query = $dbh->prepare( $sql->new_cabinet ); +$query->execute( $cname, $location); + +print " OK\n"; + +__END__ + +=pod + +=head1 NAME + +neb-cabadd - add a cabinet to a Nebulous server + +=head1 SYNOPSIS + + neb-cabadd --cname --location + [--db ] [--user ] [--pass ] [--host by creating +the appropriate set of tables. Any pre-existing tables with conflicting names +are first removed. + +=head1 OPTIONS + +=over 4 + +=item * --cname + +Symbolic name of cabinet being added. + +=item * --location + +Description of the location of the cabinet. + +=item * --db|-d + +Name of database (C) to create tables in. + +Optional if the appropriate environment variable is set. + +=item * --user|-u + +Username to authenticate with. + +Optional if the appropriate environment variable is set. + +=item * --pass|-p + +Password to authenticate with. + +Optional if the appropriate environment variable is set. + +=item * --host + +Name of host on which the database resides. + +Optional. Defaults to C. + +=head1 ENVIRONMENT + +These environment variables may be used in place of the specified command line +options. All command line option will override the corresponding environment +value. + +=over 4 + +=item * C + +Equivalent to --db|-d + +=item * C + +Equivalent to --host + +=item * C + +Equivalent to --user|-u + +=item * C + +Equivalent to --pass|-p + +=back + +=head1 CREDITS + +Josh Hoblitt and Chris Waters + +=head1 SUPPORT + +Please contact the author directly via e-mail. + +=head1 AUTHOR + +Joshua Hoblitt + +=head1 COPYRIGHT + +Copyright (C) 2005-2008 Joshua Hoblitt. All rights reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, L, L, L + +=cut Index: trunk/Nebulous-Server/bin/neb-voladm =================================================================== --- trunk/Nebulous-Server/bin/neb-voladm (revision 25010) +++ trunk/Nebulous-Server/bin/neb-voladm (revision 26292) @@ -23,4 +23,5 @@ $allocate, $available, + $mounted, $db, $dbhost, @@ -31,4 +32,5 @@ $vname, $xattr, + $cab_id, ); @@ -41,4 +43,5 @@ 'allocate=i' => \$allocate, 'available=i' => \$available, + 'mounted' => \$mounted, 'db|d=s' => \$db, 'debug' => \$debug, @@ -48,4 +51,5 @@ 'vhost=s' => \$vhost, 'vname|n=s' => \$vname, + 'cab_id|c=i' => \$cab_id, 'xattr=i' => \$xattr, ) || pod2usage( 2 ); @@ -56,6 +60,6 @@ # are we listing the volumes or updating? -if (defined $allocate or defined $available or defined $xattr) { - pod2usage( -msg => "Options: --allocate, --available, and --xattr Require options --vhost or --vname", -exitval => 2 ) +if (defined $allocate or defined $available or defined $xattr or defined $cab_id) { + pod2usage( -msg => "Options: --allocate, --available, --xattr, and --cab_id Require options --vhost or --vname", -exitval => 2 ) unless defined $vhost or defined $vname; pod2usage( -msg => "Options: --allocate, --available, and --xattr Must be 0 or 1", -exitval => 2) @@ -84,4 +88,5 @@ $set{'v.available'} = $available if defined $available; $set{'v.xattr'} = $xattr if defined $xattr; +$set{'v.cab_id'} = $cab_id if defined $cab_id; if (%set) { @@ -97,4 +102,17 @@ die $@; } + if ($mounted) { + eval { + my ($q, @bind) = sql_interp("UPDATE mountedvol AS v SET", \%set, "WHERE", \%constraint); + warn "$q\n" if $debug; + my $query = $dbh->prepare($q); + $query->execute(@bind); + $dbh->commit; + }; + if ($@) { + $dbh->rollback; + die $@; + } + } } @@ -132,4 +150,5 @@ neb-voladm [--vname ] [--vhost ] [--allocate <0|1>] [--available <0|1>] [--xattr <0|1>] + [--cab_id ] [--host ] [--db ] [--user ] [--pass ] @@ -164,4 +183,10 @@ Enables/Disables these flags on the storage volume. + +Requires either the C<--vname> or c<--vhost> options. + +=item * --cab_id + +Sets the cabinet id for this storage volume. Requires either the C<--vname> or c<--vhost> options. Index: trunk/Nebulous-Server/bin/nebdiskd =================================================================== --- trunk/Nebulous-Server/bin/nebdiskd (revision 25010) +++ trunk/Nebulous-Server/bin/nebdiskd (revision 26292) @@ -177,5 +177,6 @@ eval { - my $r_query = $dbh->prepare_cached("REPLACE INTO mountedvol SELECT *, ?, ? FROM volume WHERE mountpoint = ?"); + my $r_query = $dbh->prepare_cached("REPLACE INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, ?, ? FROM volume WHERE mountpoint = ?"); +# my $d_query = $dbh->prepare_cached("UPDATE mountedvol SET allocate = ?, available = ? WHERE mountpoint = ?"); my $d_query = $dbh->prepare_cached("DELETE FROM mountedvol WHERE mountpoint = ?"); @@ -185,5 +186,5 @@ # there may be multiple vol_ids per mountpoint but we only need to # check each mointpont once - my $query = $dbh->prepare_cached("SELECT DISTINCT mountpoint FROM volume WHERE available = 1"); + my $query = $dbh->prepare_cached("SELECT DISTINCT mountpoint FROM volume"); $query->execute; while (my $row = $query->fetchrow_hashref) { @@ -200,11 +201,13 @@ # automounter and it fails to mount my $tries = 0; + my $valid_mountpoint = 1; TEST: eval { $tries++; unless (is_mountpoint($mountpoint)) { - $log->warn("$mnt is not a valid mountpoint"); + $log->warn("$mountpoint is not a valid mountpoint ($tries)"); + $valid_mountpoint = 0; } }; - if ($@) { + if (!$valid_mountpoint) { # try is_mountpoint() again if $retry > 1 if ($tries < $retry) { @@ -212,5 +215,7 @@ goto TEST; } - $log->warn($@); + $log->warn("Removing $mountpoint from the mountedvol table ($tries > $retry)"); + # $log->warn($@); + $d_query->execute($mountpoint); next; Index: trunk/Nebulous-Server/lib/Nebulous/Server.pm =================================================================== --- trunk/Nebulous-Server/lib/Nebulous/Server.pm (revision 25010) +++ trunk/Nebulous-Server/lib/Nebulous/Server.pm (revision 26292) @@ -35,4 +35,10 @@ use constant NFS_RETRY_WAIT => 1; use constant TRANS_RETRY_WAIT => 1; + +# This is the umask hack. +umask(0002); + +# This determines how many entries from the list of volumes sorted by free space are randomized. +my $topfew_count = 3; # transaction restart/retry regex @@ -905,13 +911,15 @@ } - # In MySQL you can't select from a table your deleting rows from so + # In MySQL you can't select from a table you're deleting rows from so # we first have to get a list of instances to be removed, and then - # removed them. + # remove them. my $rows_found; { my $query = $db->prepare_cached( $sql->find_dead_instances_by_so_id ); $rows_found = $query->execute( $so_id ); - foreach my $row ($query->fetchrow_hashref) { + my $i = 0; + while (my $row = $query->fetchrow_hashref) { # remove dead instances +# $log->warn("copied: $rows_copied found: $rows_found removed: $rows_removed"); my $query = $db->prepare_cached( $sql->delete_instance_by_ins_id); $rows_removed += $query->execute( $row->{ins_id} ); @@ -919,5 +927,5 @@ $query->finish; } - +# $log->warn("copied: $rows_copied found: $rows_found removed: $rows_removed"); # sanity check die("instances inaccessible ($rows_copied) != instances removed ($rows_removed)") @@ -1692,4 +1700,108 @@ } +sub find_instances_for_cull +{ + my $self = shift; + + my $log = $self->log; + $log->debug("entered - @_"); + + my ($key, $vol_name) = validate_pos(@_, + { + type => SCALAR, + callbacks => { + 'is valid object key' => sub { $self->_is_valid_object_key($_[0]) }, + }, + }, + { + type => SCALAR|UNDEF, +# callbacks => { +# # check that the volume name requested is valid +# 'is valid volume name' => sub { +# return 1 if not defined $_[0]; +# $self->_is_valid_volume_name($_[0]) +# }, +# }, + optional => 1, + }, + ); + + my $sql = $self->sql; + +# unless ($key) { +# $log->warn("key was undefined after validate_pos(), trying again..."); +# return $self->find_instances(@_); +# } + + # vol_name overrides the key implied volume + eval { + $key = parse_neb_key($key, $vol_name); + }; + $log->logdie("$@") if $@; + $vol_name = $key->volume; + + my $db = $self->db($key); + + # the key's volume can't be validiated on input for this method so we have + # to check it after parsing the key + if (defined $vol_name + and not $self->_is_valid_volume_name($key, $key->volume)) { + if ($key->hard_volume) { + $log->logdie("$vol_name is not a valid volume name"); + } else { + $log->warn( "$vol_name is not a known volume name" ); + $vol_name = undef; + } + } + + my @locations; + eval { + my $query; + if ($vol_name) { + $query = $db->prepare_cached( $sql->get_object_instances_by_vol_name ); + # ext_id, name, available + my $rows = $query->execute($key->path, $vol_name, 1); + unless ($rows > 0) { + $query->finish; + die("no instances on storage volume or volume is not avaiable for key: $key volume: $vol_name"); + } + } else { + $query = $db->prepare_cached( $sql->get_object_instances ); + # ext_id, available + my $rows = $query->execute($key->path, 1); + unless ($rows > 0) { + $query->finish; + die("no instances available for key: $key"); + } + } + + while (my $row = $query->fetchrow_hashref) { +# my $instance_hash = { uri => $row->{ 'uri' }, +# vol_id => $row->{ 'vol_id' }, +# cab_id => $row->{ 'cab_id'} }; + my $instance = $row->{ 'uri' }; + push @locations, $row if $instance; + } + }; + if ($@) { + $db->rollback; + # handle soft volumes + if (defined $vol_name and not defined $key->hard_volume) { + $log->debug("retrying with 'any' volume"); + return $self->find_instances($key->path, 'any'); + } + $log->logdie("database error: $@"); + } + + # XXX remove this? + $log->logdie("no instances found") unless (scalar @locations); + + $log->debug("found: @locations"); + + $log->debug("leaving"); + + return \@locations; +} + sub delete_instance @@ -1984,5 +2096,6 @@ my ($key, $name, $hard_volume) = @_; - + +# $log->warn("_g_s_v: key:>$key< name:>$name< hard_vol:>$hard_volume<"); my $sql = $self->sql; my $db = $self->db($key); @@ -1996,5 +2109,5 @@ # %free, name, avaiable, allocate $rows = $query->execute(0.95, $name, 1, 1); - # XXX destinguish between non-existant and unaviable + # XXX destinguish between non-existant and unavailable unless ($rows > 0) { $query->finish; @@ -2016,5 +2129,5 @@ $query = $db->prepare_cached( $sql->get_storage_volume ); # %free, avaiable, allocate - $rows = $query->execute(0.95, 1, 1); + $rows = $query->execute(0.95, 1, 1, $topfew_count); # there has to be atleast one storage volume unless ($rows > 0) { @@ -2062,10 +2175,30 @@ my $db = $self->db($key); - my ($vol_id, $vol_host, $vol_path, $xattr); + my ($vol_id, $vol_host, $vol_path, $xattr, $forbidden_cabinet); eval { my $rows; - my $query = $db->prepare_cached( $sql->get_replication_volume_for_ext_id ); + + my $query = $db->prepare_cached( $sql->get_cabinets_for_ext_id ); + $rows = $query->execute($key->path); + unless ($rows > 0) { + $query->finish; + die("Requested key $key does not exist"); + } + if ($rows == 1) { + ($forbidden_cabinet) = $query->fetchrow_array; + unless (defined($forbidden_cabinet)) { + $forbidden_cabinet = 0; + } + $query->finish; + } + else { + $forbidden_cabinet = 0; + $query->finish; + } + + + $query = $db->prepare_cached( $sql->get_replication_volume_for_ext_id ); # ext_id, %free, avaiable, allocate - $rows = $query->execute($key->path, 0.95, 1, 1); + $rows = $query->execute($key->path, 0.95, 1, 1, $forbidden_cabinet, $topfew_count); # XXX destinguish between non-existant and unaviable unless ($rows > 0) { @@ -2175,5 +2308,5 @@ # handle "any" volume - if ($vol_name eq 'any') { + if (($vol_name eq 'any')||($vol_name eq 'any.0')) { $log->debug( "found volume name $vol_name" ); $log->debug( "leaving" ); @@ -2200,5 +2333,5 @@ $log->debug( "leaving" ); - return; + return 0; } Index: trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm =================================================================== --- trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 25010) +++ trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 26292) @@ -237,5 +237,7 @@ storage_object.so_id, uri, - available + mountedvol.available, + vol_id, + cab_id FROM storage_object JOIN instance @@ -243,11 +245,15 @@ JOIN mountedvol USING(vol_id) - WHERE ext_id = ? - AND available = ? + JOIN volume + USING(vol_id) + WHERE ext_id = ? + AND mountedvol.available = ? }, get_object_instances_by_vol_name => qq{ SELECT storage_object.so_id, - uri + uri, + vol_id, + cab_id FROM storage_object JOIN instance @@ -255,8 +261,11 @@ JOIN mountedvol USING(vol_id) - WHERE ext_id = ? - AND name = ? - AND available = ? - }, + JOIN volume + USING(vol_id) + WHERE ext_id = ? + AND mountedvol.name = ? + AND mountedvol.available = ? + }, + # volume handler get_storage_volume_by_name => qq{ SELECT @@ -275,13 +284,31 @@ LIMIT 1 }, - get_replication_volume_for_ext_id => qq{ + # volume handler + get_cabinets_for_ext_id => qq{ + SELECT DISTINCT + cab_id + FROM instance + JOIN volume ON instance.vol_id = volume.vol_id + JOIN storage_object USING(so_id) + WHERE ext_id = ? + }, + get_replication_volume_for_ext_id => qq{ + SELECT * FROM ( SELECT m.vol_id, - host, - path, - xattr, + m.host, + m.path, + m.xattr, total - used as free FROM mountedvol AS m - LEFT JOIN instance AS i + JOIN volume AS v USING(vol_id) + LEFT JOIN ( + SELECT + instance.vol_id, + so_id + FROM instance + JOIN volume + ON instance.vol_id = volume.vol_id + ) AS i ON m.vol_id = i.vol_id AND i.so_id = ( @@ -290,13 +317,19 @@ WHERE ext_id = ? ) - WHERE - i.vol_id IS NULL - AND used / total < ? - AND available = ? - AND allocate = ? - ORDER BY RAND() - LIMIT 1 - }, + WHERE + i.vol_id IS NULL + AND used / total < ? + AND m.available = ? + AND m.allocate = ? + AND ( (v.cab_id IS NULL) || + (v.cab_id != ?) ) + ORDER BY free DESC + LIMIT ?) as topfew + ORDER BY RAND() + LIMIT 1 + }, + # volume handler get_storage_volume => qq{ + SELECT * from ( SELECT vol_id, @@ -311,9 +344,15 @@ AND allocate = ? ORDER BY free DESC + LIMIT ?) as topfew + ORDER BY RAND() LIMIT 1 }, + new_cabinet => qq{ + INSERT INTO cabinet (name, location, cab_id) + VALUES (?, ?, NULL) + }, new_volume => qq{ - INSERT INTO volume (name, host, path, allocate, available, xattr, mountpoint) - VALUES (?, ?, ?, TRUE, TRUE, FALSE, ?) + INSERT INTO volume (name, host, path, allocate, available, xattr, mountpoint, cab_id) + VALUES (?, ?, ?, TRUE, TRUE, FALSE, ?, NULL) }, get_volume_by_name => qq{ @@ -331,5 +370,6 @@ v.available, v.xattr, - mountedvol.vol_id IS NOT NULL as mounted + mountedvol.vol_id IS NOT NULL as mounted, + v.cab_id FROM volume AS v LEFT JOIN mountedvol @@ -614,4 +654,15 @@ ### +CREATE TABLE cabinet ( + cab_id INT NOT NULL AUTO_INCREMENT, + name VARCHAR(255) NOT NULL, + location VARCHAR(255), + PRIMARY KEY(cab_id), + UNIQUE KEY(name), + KEY (location) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +### + CREATE TABLE volume ( vol_id INT NOT NULL AUTO_INCREMENT, @@ -623,4 +674,5 @@ xattr BOOLEAN DEFAULT FALSE, mountpoint VARCHAR(255) NOT NULL, + cab_id INT, PRIMARY KEY(vol_id), UNIQUE KEY(name), @@ -629,4 +681,5 @@ KEY(allocate), KEY(available), + FOREIGN KEY(cab_id) REFERENCES cabinet(cab_id), KEY(mountpoint(255)) ) ENGINE=innodb DEFAULT CHARSET=latin1;