Index: trunk/Nebulous-Server/bin/neb-admin =================================================================== --- trunk/Nebulous-Server/bin/neb-admin (revision 31943) +++ trunk/Nebulous-Server/bin/neb-admin (revision 32380) @@ -29,4 +29,5 @@ $pending, $balance, + $retarget, $balance_N_sources, $balance_M_destinations, @@ -49,4 +50,5 @@ 'pendingreplicate|r' => \$pending, 'pendingbalance|b' => \$balance, + 'pendingtarget|t' => \$retarget, 'balancesources|N=s' => \$balance_N_sources, 'balancedestinations|M=s' => \$balance_M_destinations, @@ -64,7 +66,7 @@ pod2usage( -msg => "--limit is meaningless without --pendingreplicate", -exitval => 2 ) - if defined $limit and not (defined $pending or defined $balance); + if defined $limit and not (defined $pending or defined $balance or defined $retarget); pod2usage( -msg => "no operation specified", -exitval => 2 ) - unless defined $pending or defined $balance; + unless defined $pending or defined $balance or defined $retarget; pod2usage( -msg => "--pendingbalance needs --balancesources and --balancedestinations options", -exitval => 2) @@ -81,4 +83,7 @@ elsif (defined $balance) { $pidfile = '/var/tmp/neb-admin.balance'; +} +elsif (defined $retarget) { + $pidfile = '/var/tmp/neb-admin.retarget'; } else { @@ -107,4 +112,6 @@ } elsif ($balance) { balance(); +} elsif ($retarget) { + retarget(); } else { die "THIS SHOULD NOT HAPPEN"; @@ -524,4 +531,144 @@ } +sub retarget +{ + my %destination; + %{ $destination{VOL} } = ('ota01' => 7, 'ota02' => 8, 'ota03' => 3, 'ota04' => 3, 'ota05' => 4, 'ota06' => 4, + 'ota10' => 50, 'ota11' => 50, 'ota12' => 6, 'ota13' => 6, 'ota14' => 7, 'ota15' => 8, 'ota16' => 15, 'ota17' => 15, + 'ota20' => 16, 'ota21' => 18, 'ota22' => 18, 'ota23' => 17, 'ota24' => 9, 'ota25' => 9, 'ota26' => 10, 'ota27' => 11, + 'ota30' => 12, 'ota31' => 52, 'ota32' => 14, 'ota33' => 14, 'ota34' => 19, 'ota35' => 16, 'ota36' => 20, 'ota37' => 12, + 'ota40' => 32, 'ota41' => 11, 'ota42' => 10, 'ota43' => 17, 'ota44' => 49, 'ota45' => 23, 'ota46' => 24, 'ota47' => 25, + 'ota50' => 26, 'ota51' => 27, 'ota52' => 28, 'ota53' => 29, 'ota54' => 30, 'ota55' => 31, 'ota56' => 51, 'ota57' => 33, + 'ota60' => 34, 'ota61' => 35, 'ota62' => 36, 'ota63' => 37, 'ota64' => 38, 'ota65' => 39, 'ota66' => 41, 'ota67' => 42, + 'ota71' => 43, 'ota72' => 44, 'ota73' => 45, 'ota74' => 46, 'ota75' => 47, 'ota76' => 53); + %{ $destination{HOST} } = ('ota01' => 'ipp010','ota02' => 'ipp011','ota03' => 'ipp006','ota04' => 'ipp006','ota05' => 'ipp007','ota06' => 'ipp007', + 'ota10' => 'ipp008','ota11' => 'ipp008','ota12' => 'ipp009','ota13' => 'ipp009','ota14' => 'ipp010','ota15' => 'ipp011','ota16' => 'ipp012','ota17' => 'ipp012', + 'ota20' => 'ipp013','ota21' => 'ipp014','ota22' => 'ipp014','ota23' => 'ipp015','ota24' => 'ipp016','ota25' => 'ipp016','ota26' => 'ipp017','ota27' => 'ipp018', + 'ota30' => 'ipp019','ota31' => 'ipp020','ota32' => 'ipp021','ota33' => 'ipp021','ota34' => 'ipp023','ota35' => 'ipp013','ota36' => 'ipp024','ota37' => 'ipp019', + 'ota40' => 'ipp025','ota41' => 'ipp018','ota42' => 'ipp017','ota43' => 'ipp015','ota44' => 'ipp027','ota45' => 'ipp028','ota46' => 'ipp029','ota47' => 'ipp030', + 'ota50' => 'ipp031','ota51' => 'ipp032','ota52' => 'ipp033','ota53' => 'ipp034','ota54' => 'ipp035','ota55' => 'ipp036','ota56' => 'ipp037','ota57' => 'ipp038', + 'ota60' => 'ipp039','ota61' => 'ipp040','ota62' => 'ipp041','ota63' => 'ipp042','ota64' => 'ipp043','ota65' => 'ipp044','ota66' => 'ipp046','ota67' => 'ipp047', + 'ota71' => 'ipp048','ota72' => 'ipp049','ota73' => 'ipp050','ota74' => 'ipp051','ota75' => 'ipp052','ota76' => 'ipp053'); + %{ $destination{NAME} } = ('ota01' => 'ipp010.0','ota02' => 'ipp011.0','ota03' => 'ipp006.0','ota04' => 'ipp006.0','ota05' => 'ipp007.0','ota06' => 'ipp007.0', + 'ota10' => 'ipp008.0','ota11' => 'ipp008.0','ota12' => 'ipp009.0','ota13' => 'ipp009.0','ota14' => 'ipp010.0','ota15' => 'ipp011.0','ota16' => 'ipp012.0','ota17' => 'ipp012.0', + 'ota20' => 'ipp013.0','ota21' => 'ipp014.0','ota22' => 'ipp014.0','ota23' => 'ipp015.0','ota24' => 'ipp016.0','ota25' => 'ipp016.0','ota26' => 'ipp017.0','ota27' => 'ipp018.0', + 'ota30' => 'ipp019.0','ota31' => 'ipp020.0','ota32' => 'ipp021.0','ota33' => 'ipp021.0','ota34' => 'ipp023.0','ota35' => 'ipp013.0','ota36' => 'ipp024.0','ota37' => 'ipp019.0', + 'ota40' => 'ipp025.0','ota41' => 'ipp018.0','ota42' => 'ipp017.0','ota43' => 'ipp015.0','ota44' => 'ipp027.0','ota45' => 'ipp028.0','ota46' => 'ipp029.0','ota47' => 'ipp030.0', + 'ota50' => 'ipp031.0','ota51' => 'ipp032.0','ota52' => 'ipp033.0','ota53' => 'ipp034.0','ota54' => 'ipp035.0','ota55' => 'ipp036.0','ota56' => 'ipp037.0','ota57' => 'ipp038.0', + 'ota60' => 'ipp039.0','ota61' => 'ipp040.0','ota62' => 'ipp041.0','ota63' => 'ipp042.0','ota64' => 'ipp043.0','ota65' => 'ipp044.0','ota66' => 'ipp046.0','ota67' => 'ipp047.0', + 'ota71' => 'ipp048.0','ota72' => 'ipp049.0','ota73' => 'ipp050.0','ota74' => 'ipp051.0','ota75' => 'ipp052.0','ota76' => 'ipp053.0'); + + if (not defined $so_id_start) { $so_id_start = 0; } + if (not defined $so_id_range) { $so_id_range = 100000; } + my $so_id_end = $so_id_start + $so_id_range; + + # XXX check if so_id_start is beyond MAX(so_id): if so, exit with exit status 10 + { + my $query = $dbh->prepare("SELECT MAX(so_id) from storage_object"); + $query->execute; + my $answer = $query->fetchrow_arrayref; + my $max_so_id = $$answer[0]; + $query->finish; + + if ($so_id_start > $max_so_id) { + print STDERR "at end of so_id range, reset please\n"; + exit 10; + } + } + + # Determine a list of first instances + my $query = $dbh->prepare( + " +SELECT S.so_id,ext_id,ins_id,uri,substr(uri,-10,5) AS class_id,vol_id,name AS vol_name,host AS vol_host FROM ( + select * FROM ( + select MAX(ins_id) AS MAXins_id, MIN(ins_id) AS MINins_id, so_id,ext_id,value AS user_copies FROM storage_object + JOIN storage_object_xattr USING(so_id) + RIGHT JOIN instance USING(so_id) + WHERE name = 'user.copies' + AND value >= 2 + AND ext_id LIKE '%ota%fits' + AND so_id >= $so_id_start + AND so_id < $so_id_end + GROUP BY so_id + ) AS V + WHERE MAXins_id != MINins_id + ) AS S + JOIN instance ON (S.MINins_id = instance.ins_id AND S.so_id = instance.so_id) JOIN volume USING(vol_id) WHERE volume.available = 1 + LIMIT $limit + OFFSET $offset +"); + $query->execute(); + + my @rows; + while (my $row = $query->fetchrow_hashref) { + push @rows, $row; + } + $query->finish; + + print STDERR "No rows found\n" unless scalar @rows; + exit unless scalar @rows; + + # compare number of responses to limit below + my $Npending = @rows; + + print "targetPending MULTI\n\n"; + + foreach my $obj (@rows) { + if (defined $verbose) { + require Data::Dumper; + print Data::Dumper::Dumper($obj); + } + + my $so_id = $obj->{so_id}; + my $key = $obj->{ext_id}; + + my $ins_id = $obj->{ins_id}; + my $uri = $obj->{uri}; + my $class_id = $obj->{class_id}; + + my $source_vol_id = $obj->{vol_id}; + my $source_name = $obj->{vol_name}; + my $source_host = $obj->{vol_host}; + + + my $destination_vol_id = $destination{VOL}{$class_id}; + my $destination_name = $destination{NAME}{$class_id}; + my $destination_host = $destination{HOST}{$class_id}; + my $destination_uri = $uri; + $destination_uri =~ s/$source_name/$destination_name/; + + my %md = ( + so_id => $so_id, + key => $key, + ins_id => $ins_id, + uri => $uri, + class_id => $class_id, + + source_vol_id => $source_vol_id, + source_name => $source_name, + source_host => $source_host, + + destination_vol_id => $destination_vol_id, + destination_name => $destination_name, + destination_host => $destination_host, + destination_uri => $destination_uri, + ); + if ($source_vol_id != $destination_vol_id) { + print_metadata("targetPending", \%md); + print "\n"; + } + } + + # use a different exit status if we hit the limit (likely more files pending) + print STDERR "Which exit: $Npending $limit\n"; + # This is a better way to do this. This should be even better-er. + if ($Npending < $limit) { + exit 0; + } + else { + exit 1; + } +} + sub removal { Index: trunk/Nebulous-Server/bin/neb-insedit =================================================================== --- trunk/Nebulous-Server/bin/neb-insedit (revision 32380) +++ trunk/Nebulous-Server/bin/neb-insedit (revision 32380) @@ -0,0 +1,280 @@ +#!/usr/bin/env perl + +# Copyright (C) 2005-2008 Joshua Hoblitt +# +# $Id: neb-insedit,v 1.2 2008-10-01 21:05:57 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.02'; + +use DBI; +use Nebulous::Server::SQL; +use Nebulous::Client; +use URI::file; +use URI; + +use Digest::MD5; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); + +my ($db, $dbhost, $dbuser, $dbpass, $key, $ins_id, $new_uri, $new_volume,$nebhost); + +$db = $ENV{'NEB_DB'} unless $db; +$dbhost = $ENV{'NEB_DBHOST'} || 'localhost'; +$dbuser = $ENV{'NEB_USER'} unless $dbuser; +$dbpass = $ENV{'NEB_PASS'} unless $dbpass; +$nebhost= $ENV{'NEB_SERVER'} unless $nebhost; + +GetOptions( + 'db|d=s' => \$db, + 'host=s' => \$dbhost, + 'pass|p=s' => \$dbpass, + 'user|u=s' => \$dbuser, + 'neb_host=s' => \$nebhost, + 'key=s' => \$key, + 'ins_id=s' => \$ins_id, + 'uri=s' => \$new_uri, + 'volume=s' => \$new_volume, +) || pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --db --user --pass --key --ins_id --uri --volume", -exitval => 2 ) + unless $db && $dbuser && $dbpass && $ins_id && $new_uri && $key && $new_volume; + +my $dbh = DBI->connect( + "DBI:mysql:database=$db:host=$dbhost", + $dbuser, + $dbpass, + { + RaiseError => 1, + PrintError => 0, + AutoCommit => 1, + }, +); + +# Connect and retrieve instance information via stat: +my $neb = Nebulous::Client->new( + proxy => "$nebhost", + ); + +die "can't connected to Nebulous Server: $nebhost" + unless defined $neb; + +my $stat = $neb->stat($key); +die "nebulous key: $key not found" unless $stat; +my $instances = $neb->find_instances($key, 'any'); +die "no instances found" unless $instances; + +my $user_copies; +# eval { +# $user_copies = $neb->getxattr($key, "user.copies"); +# }; +$user_copies = 1; +unless(defined($user_copies)) { + $user_copies = 1; +} +my @validation; +my %md5sum_uniq; +my $existing_copies = 0; +my @existance; +my @md5sums; +my @files = map {URI->new($_)->file if $_} @$instances; + +for (my $i = 0; $i <= $#files; $i++) { + if (-e $files[$i]) { + $existance[$i] = 1; + $existing_copies++; + my $fh; + open($fh,$files[$i]); + $md5sums[$i] = Digest::MD5->new->addfile($fh)->hexdigest; + $md5sum_uniq{$md5sums[$i]} = 1; + close($fh); + } + else { + $existance[$i] = 0; + $md5sums[$i] = 'NON-EXISTANT'; + $md5sum_uniq{$md5sums[$i]} = 1; + } + $validation[$i] = sprintf("% 3d %32s %s", + $existance[$i], + $md5sums[$i], + $files[$i]); +} +# Confirm everything is in a valid state +my $N_md5sums = scalar(keys %md5sum_uniq); +if ($N_md5sums != 1) { + die "Cannot modify this instance! storage_object in inconsistent state!"; +} +my $test_md5sum = $md5sums[0]; + +# Copy instance to new uri +my $destination_file = URI->new($new_uri)->file; +my $source_file = $files[0]; +system("cp $source_file $destination_file"); +system("glockfile $destination_file xcld 0"); + +my $fh; +open($fh,$destination_file); +my $new_md5sum = Digest::MD5->new->addfile($fh)->hexdigest; +close($fh); + +if ($new_md5sum ne $test_md5sum) { + die "md5sum mismatch with new copy. Exiting!"; +} + +# Update database table +my $sql = Nebulous::Server::SQL->new(); + +my $query = $dbh->prepare( $sql->get_volume_by_name ); +$query->execute( $new_volume ); +my ($vol_id, $vol_name, $vol_host, $vol_path) = $query->fetchrow_array; +$query->finish; + +print "$new_volume $vol_id $vol_name $vol_host $vol_path $new_uri $ins_id\n"; +unless (defined($vol_id)) { + die "Invalid destination volume."; +} + +$query = $dbh->prepare( $sql->update_instance_uri ); +$query->execute( $vol_id, $new_uri, $ins_id ); +$query->finish; + +unlink($source_file); +print " OK\n"; + +__END__ + +=pod + +=head1 NAME + +neb-insedit - edit the instance information + +=head1 SYNOPSIS + + neb-insedit --key --ins_id --uri --volume + [--db ] [--user ] [--pass ] [--host + +Nebulous key for the storage object associated with the instance to move. + +=item * --ins_id + +Instance identifier that will be moved + +=item * --uri + +Final location desired for this instance. Should be of the form "file://data/ipp0XX.0/nebulous/aa/aa/ins_id.nebulous:directory:tree:file". + +=item * --volume + +Volume name that this instance will be moved to. + +=item * --db|-d + +Name of database (C) to create tables in. + +Optional if the appropriate environment variable is set. + +=item * --user|-u + +Username to authenticate with. + +Optional if the appropriate environment variable is set. + +=item * --pass|-p + +Password to authenticate with. + +Optional if the appropriate environment variable is set. + +=item * --host + +Name of host on which the database resides. + +=item * --neb_host + +Location of the client side server to connect to. + +Optional. Defaults to C. + +=head1 ENVIRONMENT + +These environment variables may be used in place of the specified command line +options. All command line option will override the corresponding environment +value. + +=over 4 + +=item * C + +Equivalent to --db|-d + +=item * C + +Equivalent to --host + +=item * C + +Equivalent to --user|-u + +=item * C + +Equivalent to --pass|-p + +=item * C + +Equivalent to --neb_host + +=back + +=head1 CREDITS + +Josh Hoblitt and Chris Waters + +=head1 SUPPORT + +Please contact the author directly via e-mail. + +=head1 AUTHOR + +Joshua Hoblitt + +=head1 COPYRIGHT + +Copyright (C) 2005-2008 Joshua Hoblitt. All rights reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, L, L, L + +=cut