Index: /trunk/Nebulous-Server/Changes =================================================================== --- /trunk/Nebulous-Server/Changes (revision 24539) +++ /trunk/Nebulous-Server/Changes (revision 24540) @@ -24,4 +24,7 @@ - add nebdiskd mountpoint test retyring - rework delete_instance() to avoid requiring an index on instance.uri + - nebdiskd bug fixes: fix debug mode outpoint going to mail, remove db + passwd requirement (not all dbs require a password param) + - completely rework how mountedvol is populated, drop mount table 0.16 Index: /trunk/Nebulous-Server/bin/nebdiskd =================================================================== --- /trunk/Nebulous-Server/bin/nebdiskd (revision 24539) +++ /trunk/Nebulous-Server/bin/nebdiskd (revision 24540) @@ -37,5 +37,5 @@ GetOptions( - 'dbhost|h=s' => \$dbhost, + 'dbhost|H=s' => \$dbhost, 'dbpass|p=s' => \$dbpass, 'dbuser|u=s' => \$dbuser, @@ -47,5 +47,5 @@ 'retry=i' => \$retry, 'stop|s' => \$stop, - 'user=s' => \$user, + 'user|U=s' => \$user, 'verbose|v' => \$verbose, ) || pod2usage( 2 ); @@ -61,5 +61,5 @@ $dbhost ||= $c->get_dbhost || $ENV{'NEB_DBHOST'} || 'localhost'; $dbuser ||= $c->get_dbuser || $ENV{'NEB_USER'}; -$dbpass ||= $c->get_dbpass || $ENV{'NEB_PASS'}; +$dbpass ||= $c->get_dbpass || $ENV{'NEB_PASS'} || undef; $pidfile ||= $c->get_pidfile || "/var/tmp/nebdiskd"; $user ||= $c->get_user || $<; # user @@ -85,6 +85,6 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --db --user --pass", -exitval => 2 ) - unless $db && $dbuser && $dbpass; +pod2usage( -msg => "Required options: --db --dbuser", -exitval => 2 ) + unless $db && $dbuser; # start up logging @@ -111,5 +111,5 @@ log4perl.appender.Mailer.subject = nebdiskd alert log4perl.appender.Mailer.buffered = 0 - log4perl.appender.AppError.Filter= MatchWarn + log4perl.appender.Mailer.Filter= MatchWarn log4perl.appender.Mailer.layout = Log::Log4perl::Layout::PatternLayout log4perl.appender.Mailer.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n @@ -163,5 +163,5 @@ my $dbhost = $p{dbhost} or return; my $dbuser = $p{dbuser} or return; - my $dbpass = $p{dbpass} or return; +# my $dbpass = $p{dbpass} or return; my $poll_interval = $p{poll_interval} || 60; my $debug = $p{debug} || 0; @@ -177,14 +177,16 @@ eval { - my $r_query = $dbh->prepare_cached("REPLACE INTO mount VALUES(?, ?, ?)"); - my $d_query = $dbh->prepare_cached("DELETE FROM mount, mountedvol WHERE mountpoint = ?"); + my $r_query = $dbh->prepare_cached("REPLACE INTO mountedvol SELECT *, ?, ? FROM volume WHERE mountpoint = ?"); + my $d_query = $dbh->prepare_cached("DELETE FROM mountedvol WHERE mountpoint = ?"); # get list of mount points my $mounts = []; { - my $query = $dbh->prepared_cached("SELECT mountpoint FROM volume"); + # there may be multiple vol_ids per mountpoint but we only need to + # check each mointpont once + my $query = $dbh->prepare_cached("SELECT DISTINCT mountpoint FROM volume"); $query->execute; while (my $row = $query->fetchrow_hashref) { - push @$mounts, $row->{'mountpoint'}; + push @$mounts, $row; } $query->finish; @@ -193,5 +195,6 @@ # determine valid mountpoints foreach my $mnt (@$mounts) { - $log->debug("checking $mnt"); + my $mountpoint = $mnt->{'mountpoint'}; + $log->debug("checking $mountpoint"); # this /SHOULD/ fail if the mount point is handled by the # automounter and it fails to mount @@ -199,5 +202,5 @@ TEST: eval { $tries++; - unless (is_mountpoint($mnt)) { + unless (is_mountpoint($mountpoint)) { $log->warn("$mnt is not a valid mountpoint"); } @@ -206,9 +209,9 @@ # try is_mountpoint() again if $retry > 1 if ($tries < $retry) { - $log->warn("retrying test of $mnt"); + $log->warn("retrying test of $mountpoint"); goto TEST; } $log->warn($@); - $d_query->execute($mnt); + $d_query->execute($mountpoint); next; } @@ -217,17 +220,17 @@ # we determine if it's a valid mountpoint incase # is_mountpoint() invokes the automounter - my $dev_info = df($mnt, 1024); + my $dev_info = df($mountpoint, 1024); unless (defined $dev_info) { - $log->error("can't find device info for $mnt"); + $log->error("can't find device info for $mountpoint"); next; } # find vol_id(s) for mountpoint - $r_query->execute($mnt, @$dev_info{qw( blocks used )}); - $log->debug("adding $mnt to db"); + $r_query->execute(@$dev_info{qw( blocks used )}, $mountpoint); + $log->debug("adding $mountpoint to db"); } - $dbh->do("call getmountedvol()"); +# $dbh->do("call getmountedvol()"); $dbh->commit; @@ -263,8 +266,9 @@ my %p = @_; + # $p{dbpass} may be undef; + return unless defined $p{db} and defined $p{dbhost} - and defined $p{dbuser} - and defined $p{dbpass}; + and defined $p{dbuser}; my $sql = Nebulous::Server::SQL->new; @@ -372,5 +376,8 @@ =head1 SYNOPSIS - nebdiskd [--db ] [--user ] [--pass ] [--debug] [--pidfile ] [--stop] [--restart] [--verbose] + nebdiskd [--db|-D ] [--dbhost|-H + [--dbuser|-u ] [--dbpass|-p ] [--debug|-d] + [--user|-U ] [--group|-g ] [--pidfile ] + [--retry ] [--stop|-s] [--restart|-r] [--verbose|-v] =head1 DESCRIPTION @@ -383,5 +390,5 @@ =over 4 -=item * --db|-d +=item * --db|-D Name of database (C) to write too. @@ -390,5 +397,5 @@ variable is set. -=item * --user|-u +=item * --dbuser|-u Username to authenticate with. @@ -397,12 +404,19 @@ variable is set. -=item * --pass|-p +=item * --dbpass|-p Password to authenticate with. + +Optional if defined in the F<.nebdiskdrc> file or the appropriate environment +variable is set or if the database does not require a password. + +=item * --dbhost|-H + +Database host to connect to. Optional if defined in the F<.nebdiskdrc> file or the appropriate environment variable is set. -=item * --debug +=item * --debug|-d This flag prevents the program from "daemonizing" so output can be sent to @@ -425,5 +439,10 @@ instance. -=item * --verbose|-r +=item * --retry + +The number of times to test a mountpoint for "mountedness" before giving up on +it. + +=item * --verbose|-v Turns on informational/debugging messages to be sent to the @@ -431,4 +450,12 @@ with C<--debug>. +=item * --user|-U + +user account to run daemon as. + +=item * --group|-g + +group to run daemon as. + =back @@ -443,13 +470,13 @@ =item * C -Equivalent to --db|-d +Equivalent to --db|-D =item * C -Equivalent to --user|-u +Equivalent to --dbuser|-u =item * C -Equivalent to --pass|-p +Equivalent to --dbpass|-p =back @@ -467,8 +494,4 @@ dbpass: '@neb@' dbuser: nebulous - mounts: - - /mnt - - /tmp - - /usr pidfile: /var/tmp/nebdiskd poll_interval: 5 @@ -480,10 +503,4 @@ =over 4 -=item * C - -A list of "paths" to C before mounted volumes are polled. The propose -of this option is to attempt to keep "automounted" volumes mounted while this -program is running. - This value may be omitted or left blank. @@ -493,4 +510,8 @@ This value may be omitted or left blank. The default value is C<60>s. + +=item * C + +Same as C<--retry>. =back Index: /trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm =================================================================== --- /trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 24539) +++ /trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 24540) @@ -396,5 +396,5 @@ }, get_mounted_volumes => qq{ - SELECT * FROM mountedvol ORDER BY host, name + SELECT mountpoint, total, used, vol_id, name, host, path, allocate, available, xattr FROM mountedvol ORDER BY host, name }, ); @@ -423,7 +423,6 @@ DROP TABLE IF EXISTS lock_record; DROP TABLE IF EXISTS volume; -DROP TABLE IF EXISTS mount; +DROP TABLE IF EXISTS mountedvol; DROP TABLE IF EXISTS log; -DROP TABLE IF EXISTS mountedvol; DROP TABLE IF EXISTS directory; DROP PROCEDURE IF EXISTS getmountedvol; @@ -518,8 +517,8 @@ host VARCHAR(255) NOT NULL, path VARCHAR(255) UNIQUE NOT NULL, - mountpoint VARCHAR(255) NOT NULL, allocate BOOLEAN DEFAULT FALSE, available BOOLEAN DEFAULT FALSE, xattr BOOLEAN DEFAULT FALSE, + mountpoint VARCHAR(255) NOT NULL, PRIMARY KEY(vol_id), KEY(host(16)), @@ -527,4 +526,30 @@ KEY(allocate), KEY(available) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +### + +CREATE TABLE mountedvol( + vol_id INT UNIQUE NOT NULL, + FOREIGN KEY(vol_id) REFERENCES volume(vol_id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + host VARCHAR(255) NOT NULL, + path VARCHAR(255) NOT NULL, + FOREIGN KEY(path) REFERENCES volume(path) ON DELETE CASCADE, + allocate BOOLEAN DEFAULT FALSE, + available BOOLEAN DEFAULT FALSE, + xattr BOOLEAN DEFAULT FALSE, + mountpoint VARCHAR(255) NOT NULL, + FOREIGN KEY(mountpoint) REFERENCES volume(mountpoint) ON DELETE CASCADE, + total BIGINT NOT NULL, + used BIGINT NOT NULL, + PRIMARY KEY(vol_id), + KEY(name), + KEY(host), + KEY(path), + KEY(allocate), + KEY(available), + KEY(xattr), + KEY(mountpoint(255)) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -548,13 +573,4 @@ ### -CREATE TABLE mount ( - mountpoint VARCHAR(255) NOT NULL, - total BIGINT NOT NULL, - used BIGINT NOT NULL, - PRIMARY KEY(mountpoint) -) ENGINE=innodb DEFAULT CHARSET=latin1; - -### - CREATE TABLE log ( timestamp TIMESTAMP, @@ -564,30 +580,4 @@ message VARCHAR(2048) NOT NULL, PRIMARY KEY(timestamp) -) ENGINE=innodb DEFAULT CHARSET=latin1; - -### - -CREATE TABLE mountedvol( - mountpoint VARCHAR(255) NOT NULL, - FOREIGN KEY(mountpoint) REFERENCES mount(mountpoint) ON DELETE CASCADE, - total BIGINT NOT NULL, - used BIGINT NOT NULL, - vol_id INT NOT NULL, - FOREIGN KEY(vol_id) REFERENCES volume(vol_id) ON DELETE CASCADE, - name VARCHAR(255) NOT NULL, - host VARCHAR(255) NOT NULL, - path VARCHAR(255) NOT NULL, - FOREIGN KEY(path) REFERENCES volume(path) ON DELETE CASCADE, - allocate BOOLEAN DEFAULT FALSE, - available BOOLEAN DEFAULT FALSE, - xattr BOOLEAN DEFAULT FALSE, - PRIMARY KEY(mountpoint), - KEY(vol_id), - KEY(name), - KEY(host), - KEY(path), - KEY(allocate), - KEY(available), - KEY(xattr) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /trunk/Nebulous-Server/lib/Test/Nebulous.pm =================================================================== --- /trunk/Nebulous-Server/lib/Test/Nebulous.pm (revision 24539) +++ /trunk/Nebulous-Server/lib/Test/Nebulous.pm (revision 24540) @@ -55,31 +55,38 @@ # node01/node02: allocate = TRUE - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (1, 'node01', 'node01', ?, TRUE, TRUE) }, undef, $dir1); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir1); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (1, 'node01', 'node01', ?, '/', TRUE, TRUE) }, undef, $dir1); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 1); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir1); - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (2, 'node02', 'node02', ?, TRUE, TRUE) }, undef, $dir2); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e8) }, undef, $dir2); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (2, 'node02', 'node02', ?, '/', TRUE, TRUE) }, undef, $dir2); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e8 FROM volume WHERE vol_id = ? }, undef, 2); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e8) }, undef, $dir2); # node03: allocate = TRUE - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (3, 'node03', 'node03', ?, TRUE, TRUE) }, undef, $dir3); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e8) }, undef, $dir3); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (3, 'node03', 'node03', ?, '/', TRUE, TRUE) }, undef, $dir3); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e8 FROM volume WHERE vol_id = ? }, undef, 3); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e8) }, undef, $dir3); # node04: allocate = FALSE, available = FALSE - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (4, 'node04', 'node04', ?, FALSE, FALSE) }, undef, $dir4); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir4); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (4, 'node04', 'node04', ?, '/', FALSE, FALSE) }, undef, $dir4); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 4); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir4); # node05: allocate = FALSE, available = TRUE - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (5, 'node05', 'node05', ?, FALSE, TRUE) }, undef, $dir5); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir5); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (5, 'node05', 'node05', ?, '/', FALSE, TRUE) }, undef, $dir5); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 5); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir5); # node06: allocate = TRUE, available = FALSE - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (6, 'node06', 'node06', ?, TRUE, FALSE) }, undef, $dir6); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir6); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (6, 'node06', 'node06', ?, '/', TRUE, FALSE) }, undef, $dir6); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 6); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir6); # node07: full - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (7, 'node07', 'node07', ?, TRUE, TRUE) }, undef, $dir7); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e10) }, undef, $dir7); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (7, 'node07', 'node07', ?, '/', TRUE, TRUE) }, undef, $dir7); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e10 FROM volume WHERE vol_id = ? }, undef, 7); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e10) }, undef, $dir7); - $dbh->do(qq{ call getmountedvol() }); +# $dbh->do(qq{ call getmountedvol() }); }