Index: trunk/Nebulous/bin/neb-addvol =================================================================== --- trunk/Nebulous/bin/neb-addvol (revision 16315) +++ (revision ) @@ -1,177 +1,0 @@ -#!/usr/bin/env perl - -# Copyright (C) 2005-2008 Joshua Hoblitt -# -# $Id: neb-addvol,v 1.8 2008-02-05 23:14:22 eugene Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use vars qw( $VERSION ); -$VERSION = '0.01'; - -use DBI; -use Nebulous::Server::SQL; -use URI::file; -use URI; - -use Getopt::Long qw( GetOptions :config auto_help auto_version ); -use Pod::Usage qw( pod2usage ); - -my ($db, $dbuser, $dbpass, $volume, $uri); - -$db = $ENV{'NEB_DB'} unless $db; -$dbuser = $ENV{'NEB_USER'} unless $dbuser; -$dbpass = $ENV{'NEB_PASS'} unless $dbpass; - -GetOptions( - 'db=s' => \$db, - 'user=s' => \$dbuser, - 'pass=s' => \$dbpass, - 'volume=s' => \$volume, - 'uri|u=s' => \$uri, -) || pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --db --user --pass", -exitval => 2 ) - unless $db && $dbuser && $dbpass && $volume && $uri; - -my $dbh = DBI->connect( - "DBI:mysql:database=$db:host=localhost", - $dbuser, - $dbpass, - { - RaiseError => 1, - PrintError => 0, - AutoCommit => 1, - }, -); - -my $sql = Nebulous::Server::SQL->new(); - -print "Checking URI..."; - -my $path = URI->new($uri)->path; -unless (-d $path) { - die "path: $path dirived from URI: $uri does not exist"; -} - -print " OK\n"; - -print "Adding volume..."; - -my $query = $dbh->prepare( $sql->new_volume ); -$query->execute( $volume, $path ); - -print " OK\n"; - -__END__ - -=pod - -=head1 NAME - -neb-addvol - add a storage volume to a Nebulous server - -=head1 SYNOPSIS - - neb-addvol --volume --uri - [--db ] [--user ] [--pass ] - -=head1 DESCRIPTION - -This program initialize a database for use by L by creating -the appropriate set of tables. Any pre-existing tables with conflicting names -are first removed. - -=head1 OPTIONS - -=over 4 - -=item * --volume - -Symbolic name of volume being added. - -=item * --uri|-u - -URI to the volume being added. - -=item * --db|-d - -Name of database (C) to create tables in. - -Optional if the appropriate environment variable is set. - -=item * --user|-u - -Username to authenticate with. - -Optional if the appropriate environment variable is set. - -=item * --pass|-p - -Password to authenticate with. - -Optional if the appropriate environment variable is set. - -=back - -=head1 ENVIRONMENT - -These environment variables may be used in place of the specified command line -options. All command line option will override the corresponding environment -value. - -=over 4 - -=item * C - -Equivalent to --db|-d - -=item * C - -Equivalent to --user|-u - -=item * C - -Equivalent to --pass|-p - -=back - -=head1 CREDITS - -Just me, myself, and I. - -=head1 SUPPORT - -Please contact the author directly via e-mail. - -=head1 AUTHOR - -Joshua Hoblitt - -=head1 COPYRIGHT - -Copyright (C) 2005-2008 Joshua Hoblitt. All rights reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; either version 2 of the License, or (at your option) any later -version. - -This program is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 Temple -Place - Suite 330, Boston, MA 02111-1307, USA. - -The full text of the license can be found in the L Pod as supplied -with Perl 5.8.1 and later. - -=head1 SEE ALSO - -L, L, L, L - -=cut Index: trunk/Nebulous/bin/neb-fsck =================================================================== --- trunk/Nebulous/bin/neb-fsck (revision 16315) +++ (revision ) @@ -1,205 +1,0 @@ -#!/usr/bin/env perl - -# Copyright (C) 2005-2008 Joshua Hoblitt -# -# $Id: neb-fsck,v 1.1 2008-01-30 00:19:34 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use vars qw( $VERSION ); -$VERSION = '0.01'; - -use DBI; -use Nebulous::Server::SQL; -use Nebulous::Client; -use URI; - -use Getopt::Long qw( GetOptions :config auto_help auto_version ); -use Pod::Usage qw( pod2usage ); - -my ($db, $dbuser, $dbpass, $server); - -$db = $ENV{'NEB_DB'} unless $db; -$dbuser = $ENV{'NEB_USER'} unless $dbuser; -$dbpass = $ENV{'NEB_PASS'} unless $dbpass; -$server = $ENV{'NEB_SERVER'} unless $server; - -GetOptions( - 'db=s' => \$db, - 'user=s' => \$dbuser, - 'pass=s' => \$dbpass, - 'server|s=s' => \$server, -) || pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --server", -exitval => 2 ) - unless $server; -pod2usage( -msg => "Required options: --db --user --pass", -exitval => 2 ) - unless $db && $dbuser && $dbpass; - -my $neb = Nebulous::Client->new( - proxy => "$server", -); - -die "can't connected to Nebulous Server: $server" - unless defined $neb; - -my $dbh = DBI->connect( - "DBI:mysql:database=$db:host=localhost", - $dbuser, - $dbpass, - { - RaiseError => 1, - PrintError => 0, - AutoCommit => 1, - }, -); - -my $sql = Nebulous::Server::SQL->new(); - -# so_id, ext_id, instances, available_instances, need_recovery, recoverable -my $query = $dbh->prepare( $sql->find_objects_with_unavailable_instances ); -$query->execute; - -my @rows; -while (my $row = $query->fetchrow_hashref) { - push @rows, $row; -} -$query->finish; - -foreach my $obj (@rows) { - my $so_id = $obj->{so_id}; - my $key = $obj->{ext_id}; - my $has = $obj->{instances}; - my $available = $obj->{instances_available}; - my $need = $obj->{need_recovery}; - my $recoverable = $obj->{recoverable}; - - # objects with only a single instance that are offline are unrecoverable so - # we don't need to handle that special case - next unless $need; - unless ($recoverable) { - warn "so_id: $so_id key: \'$key\' ", - "can not be recovered - no available instances\n"; - next; - } - - $available ||= 0; - next unless $available < 2; - - warn "so_id: $so_id key: \'$key\' ", - " has only 1 instance avaiable -- replicating"; - - $neb->replicate($obj->{ext_id}) - or warn "so_id: $so_id key: \'$key\' failed to replicate"; -} - -__END__ - -=pod - -=head1 NAME - -neb-fsck - check and repair Nebulous volumes - -=head1 SYNOPSIS - - neb-fsck [--db ] [--user ] [--pass ] - [--server ] - -=head1 DESCRIPTION - -=head1 OPTIONS - -=over 4 - -=item * --server|-s - -URL of the Nebulous server to connect to. - -Optional if the appropriate environment variable is set. - -=item * --db|-d - -Name of database (C) to create tables in. - -Optional if the appropriate environment variable is set. - -=item * --user|-u - -Username to authenticate with. - -Optional if the appropriate environment variable is set. - -=item * --pass|-p - -Password to authenticate with. - -Optional if the appropriate environment variable is set. - -=back - -=head1 ENVIRONMENT - -These environment variables may be used in place of the specified command line -options. All command line option will override the corresponding environment -value. - -=over 4 - -=item * C - -Equivalent to --server|-s - -=item * C - -Equivalent to --db|-d - -=item * C - -Equivalent to --user|-u - -=item * C - -Equivalent to --pass|-p - -=back - -=head1 CREDITS - -Just me, myself, and I. - -=head1 SUPPORT - -Please contact the author directly via e-mail. - -=head1 AUTHOR - -Joshua Hoblitt - -=head1 COPYRIGHT - -Copyright (C) 2005-2008 Joshua Hoblitt. All rights reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; either version 2 of the License, or (at your option) any later -version. - -This program is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 Temple -Place - Suite 330, Boston, MA 02111-1307, USA. - -The full text of the license can be found in the L Pod as supplied -with Perl 5.8.1 and later. - -=head1 SEE ALSO - -L, L, L, L - -=cut Index: trunk/Nebulous/bin/neb-initdb =================================================================== --- trunk/Nebulous/bin/neb-initdb (revision 16315) +++ (revision ) @@ -1,162 +1,0 @@ -#!/usr/bin/env perl - -# Copyright (C) 2005-2007 Joshua Hoblitt -# -# $Id: neb-initdb,v 1.7 2007-04-28 01:19:59 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use vars qw( $VERSION ); -$VERSION = '0.02'; - -use DBI; -use Nebulous::Server::SQL; - -use Getopt::Long qw( GetOptions :config auto_help auto_version ); -use Pod::Usage qw( pod2usage ); - -my ($db, $dbuser, $dbpass); - -$db = $ENV{'NEB_DB'} unless $db; -$dbuser = $ENV{'NEB_USER'} unless $dbuser; -$dbpass = $ENV{'NEB_PASS'} unless $dbpass; - -GetOptions( - 'db=s' => \$db, - 'user=s' => \$dbuser, - 'pass=s' => \$dbpass, -) || pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --db --user --pass", -exitval => 2 ) - unless $db && $dbuser && $dbpass; - -my $dbh = DBI->connect( - "DBI:mysql:database=$db:host=localhost", - $dbuser, - $dbpass, - { - RaiseError => 1, - PrintError => 0, - AutoCommit => 1, - }, -); - -my $sql = Nebulous::Server::SQL->new(); - -print "Dropping any existing tables..."; - -foreach my $statement (@{ $sql->get_db_clear }) { - $dbh->do( $statement ); -} - -print " OK\nCreating new tables..."; - -foreach my $statement (@{ $sql->get_db_schema }) { - $dbh->do( $statement ); -} - -print " OK\n"; - -__END__ - -=pod - -=head1 NAME - -neb-initdb - initialize a Nebulous server database - -=head1 SYNOPSIS - - neb-initdb [--db ] [--user ] [--pass ] - -=head1 DESCRIPTION - -This program initialize a database for use by L by creating -the appropriate set of tables. Any pre-existing tables with conflicting names -are first removed. - -=head1 OPTIONS - -=over 4 - -=item * --db|-d - -Name of database (C) to create tables in. - -Optional if the appropriate environment variable is set. - -=item * --user|-u - -Username to authenticate with. - -Optional if the appropriate environment variable is set. - -=item * --pass|-p - -Password to authenticate with. - -Optional if the appropriate environment variable is set. - -=back - -=head1 ENVIRONMENT - -These environment variables may be used in place of the specified command line -options. All command line option will override the corresponding environment -value. - -=over 4 - -=item * C - -Equivalent to --db|-d - -=item * C - -Equivalent to --user|-u - -=item * C - -Equivalent to --pass|-p - -=back - -=head1 CREDITS - -Just me, myself, and I. - -=head1 SUPPORT - -Please contact the author directly via e-mail. - -=head1 AUTHOR - -Joshua Hoblitt - -=head1 COPYRIGHT - -Copyright (C) 2005-2007 Joshua Hoblitt. All rights reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; either version 2 of the License, or (at your option) any later -version. - -This program is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 Temple -Place - Suite 330, Boston, MA 02111-1307, USA. - -The full text of the license can be found in the L Pod as supplied -with Perl 5.8.1 and later. - -=head1 SEE ALSO - -L, L, L, L - -=cut Index: trunk/Nebulous/bin/nebdiskd =================================================================== --- trunk/Nebulous/bin/nebdiskd (revision 16315) +++ (revision ) @@ -1,412 +1,0 @@ -#!/usr/bin/env perl - -# Copyright (C) 2007 Joshua Hoblitt -# -# $Id: nebdiskd,v 1.4 2007-05-04 23:36:46 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use vars qw( $VERSION ); -$VERSION = '0.02'; - -use Config::YAML; -use DBI; -use File::Spec; -use Nebulous::Server::SQL; -use Net::Server::Daemonize qw( daemonize unlink_pid_file ); -use Sys::Statistics::Linux::DiskUsage; - -use Getopt::Long qw( GetOptions :config auto_help auto_version ); -use Pod::Usage qw( pod2usage ); - -my ($debug, $db, $dbuser, $dbpass, $pidfile, $stop, $restart, $verbose); -GetOptions( - 'debug' => \$debug, - 'db|d=s' => \$db, - 'user|u=s' => \$dbuser, - 'pass|p=s' => \$dbpass, - 'pidfile=s' => \$pidfile, - 'stop|s' => \$stop, - 'restart|r' => \$restart, - 'verbose|v' => \$verbose, -) || pod2usage( 2 ); - -my $rcfile = "$ENV{HOME}/.nebdiskrc"; -$rcfile = File::Spec->canonpath($rcfile); - -my $c = read_rcfile($rcfile); - -$db ||= $c->get_db || $ENV{'NEB_DB'}; -$dbuser ||= $c->get_dbuser || $ENV{'NEB_USER'}; -$dbpass ||= $c->get_dbpass || $ENV{'NEB_PASS'}; -$pidfile ||= $c->get_pidfile || "/var/tmp/nebdiskd"; -my $mounts = $c->get_mounts; -my $poll_interval = $c->get_poll_interval || 5; - -if ($restart || $stop) { - my $status = kill_pid_file($pidfile); - exit($status) if $stop; -} - -$c->set_db($db); -$c->set_dbuser($dbuser); -$c->set_dbpass($dbpass); -$c->set_pidfile($pidfile); -$c->set_poll_interval($poll_interval); -$c->write; - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --db --user --pass", -exitval => 2 ) - unless $db && $dbuser && $dbpass; - -daemonize( - $<, # User - $), # Group - $pidfile, # PID file -) unless $debug; - -$SIG{TERM} = sub { unlink_pid_file($pidfile); exit(); }; -$SIG{INT} = $SIG{TERM}; -$SIG{HUP} = sub { $c = read_rcfile($rcfile) }; - -my $dbh = setup_db( - db => $db, - dbuser => $dbuser, - dbpass => $dbpass, - ); - -poll_mounts( - dbh => $dbh, - poll_interval => $poll_interval, - debug => $debug, - ) or die "poll_mounts() should not have returned"; - -sub poll_mounts -{ - my %p = @_; - - my $dbh = $p{dbh} or return; - my $poll_interval = $p{poll_interval} || 60; - my $debug = $p{debug} || 0; - - my $lxs = Sys::Statistics::Linux::DiskUsage->new; - - while (1) { - # list of mount points to stat so that the automounter will mount these - # volumes if they have been unmounted. - stat_dirs($mounts); - - eval { - my $query = $dbh->prepare_cached("INSERT INTO mount VALUES(?, ?, ?)"); - $dbh->do("DELETE FROM mount"); - - print "flushed mount table\n" if $debug; - - my $stats = $lxs->get; - foreach my $dev (keys %$stats) { - my $mnt = $stats->{$dev}; - my %mount; - $query->execute(@$mnt{qw( mountpoint total usage )}); - print "adding $mnt->{mountpoint} to db\n" if $debug; - } - - $dbh->do("call getmountedvol()"); - - $dbh->commit; - }; - if ($@) { - $db->rollback; - warn $@; - } - - sleep $poll_interval; - } -} - -sub read_rcfile -{ - my $rcfile = shift; - - return unless defined $rcfile; - - if (!-f $rcfile) { - open(my $fh, '>', $rcfile) or die "can't open file: $!"; - close($fh) or die "can't close file:$!"; - } - - return Config::YAML->new( - config => $rcfile, - output => $rcfile, - ); -} - - -sub setup_db -{ - my %p = @_; - - return unless defined $p{db} - and defined $p{dbuser} - and defined $p{dbpass}; - - my $sql = Nebulous::Server::SQL->new; - - my $dbh = DBI->connect( - "DBI:mysql:database=$p{db}:host=localhost", - $p{dbuser}, - $p{dbpass}, - { - RaiseError => 1, - PrintError => 1, - AutoCommit => 0, - }, - ); - - eval { - $dbh->do( $sql->set_transaction_model ); - $dbh->commit; - }; - if ($@) { - $db->rollback; - die $@; - } - - return $dbh; -} - - -sub stat_dirs -{ - my $mounts = shift; - - return unless defined $mounts; - - foreach my $path (@$mounts) { - stat File::Spec->canonpath($path) or warn "can not stat path: $path"; - print "stated $path\n" if $debug; - } -} - -# -# This function, originally named check_pid_file() was copied from -# Net::Server::Daemonize '0.05' licensed under the Perl license. It has been -# renamed and modified to kill off process who's PID value is stored in pidfile. -# - -sub kill_pid_file { - my $pid_file = shift; - - ### no pid_file = return success - return 1 unless -e $pid_file; - - ### get the currently listed pid - if( ! open(_PID,$pid_file) ){ - die "Couldn't open existant pid_file \"$pid_file\" [$!]\n"; - } - my $_current_pid = <_PID>; - close _PID; - my $current_pid = $_current_pid =~ /^(\d{1,10})/ ? $1 : die "Couldn't find pid in existing pid_file"; - - my $exists = undef; - - ### try a proc file system - if( -d '/proc' ) { - $exists = -e "/proc/$current_pid"; - - ### try ps - #}elsif( -x '/bin/ps' ){ # not as portable - # the ps command itself really isn't portable - # this follows BSD syntax ps (BSD's and linux) - # this will fail on Unix98 syntax ps (Solaris, etc) - }elsif( `ps h o pid p $$` =~ /^\s*$$\s*$/ ){ # can I play ps on myself ? - $exists = `ps h o pid p $current_pid`; - - } - - ### running process exists, ouch - if( $exists ){ - - if( $current_pid == $$ ){ - warn "Pid_file created by this same process. Doing nothing.\n"; - return 1; - }else{ - kill 'TERM', $current_pid - or die "Failed to signal process ($current_pid)"; - unlink $pid_file || die "Couldn't remove pid_file \"$pid_file\" [$!]\n"; - return 1; - - } - } -} - -__END__ - -=pod - -=head1 NAME - -nebdiskd - Nebulous mounted volume capacity monitoring daemon - -=head1 SYNOPSIS - - nebdiskd [--db ] [--user ] [--pass ] [--debug] [--pidfile ] [--stop] [--restart] [--verbose] - -=head1 DESCRIPTION - -This program looks for mounted volumes, at a configurable interval, and records -statistics about them into a database. - -=head1 OPTIONS - -=over 4 - -=item * --db|-d - -Name of database (C) to write too. - -Optional if defined in the F<.nebdiskdrc> file or the appropriate environment -variable is set. - -=item * --user|-u - -Username to authenticate with. - -Optional if defined in the F<.nebdiskdrc> file or the appropriate environment -variable is set. - -=item * --pass|-p - -Password to authenticate with. - -Optional if defined in the F<.nebdiskdrc> file or the appropriate environment -variable is set. - -=item * --debug - -This flag prevents the program from "daemonizing" so output can be sent to -C/C and a debugger used on the running process. - -=item * --pidfile - -Filename to log the running daemon's PID too. - -Optional, if this option is omitted and no value is defined in the -F<.nebdiskrc> then no pidfile is created. - -=item * --stop|-s - -Uses the pidfile to kill an already running daemon. - -=item * --restart|-r - -Uses the pidfile to kill an already running daemon and then starts a new -instance. - -=item * --verbose|-r - -Turns on informational/debugging messages to be sent to the -C/C. This option is probably not very usefully unless combined -with C<--debug>. - -=back - -=head1 ENVIRONMENT - -These environment variables may be used in place of the specified command line -options. All command line option will override the corresponding environment -value. - -=over 4 - -=item * C - -Equivalent to --db|-d - -=item * C - -Equivalent to --user|-u - -=item * C - -Equivalent to --pass|-p - -=back - -=head1 RCFILE - -This program will attempt to read an C from F<$HOME/.nebdiskrc> and if -found will use it's values to override program defaults. Please not that the -precedence for values is: command line, rcfile, environment variables. - -The format of the C is in L format and uses the following values: - - --- - db: nebulous - dbpass: '@neb@' - dbuser: nebulous - mounts: - - /mnt - - /tmp - - /usr - pidfile: /var/tmp/nebdiskd - poll_interval: 5 - -The values C, C, C, and C have the same semantics -as their command line and/or environment variable equivalents. - -=over 4 - -=item * C - -A list of "paths" to C before mounted volumes are polled. The propose -of this option is to attempt to keep "automounted" volumes mounted while this -program is running. - -This value may be omitted or left blank. - -=item * C - -The number of seconds to wait between checks for mounted volumes. - -This value may be omitted or left blank. The default value is C<60>s. - -=back - -=head1 CREDITS - -Just me, myself, and I. - -=head1 SUPPORT - -Please contact the author directly via e-mail. - -=head1 AUTHOR - -Joshua Hoblitt - -=head1 COPYRIGHT - -Copyright (C) 2007 Joshua Hoblitt. All rights reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; either version 2 of the License, or (at your option) any later -version. - -This program is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 Temple -Place - Suite 330, Boston, MA 02111-1307, USA. - -The full text of the license can be found in the L Pod as supplied -with Perl 5.8.1 and later. - -=head1 SEE ALSO - -L, L, L, L, L - -=cut