Index: trunk/tools/eam/rebalance/neb_check_trash.pl
===================================================================
--- trunk/tools/eam/rebalance/neb_check_trash.pl	(revision 41384)
+++ trunk/tools/eam/rebalance/neb_check_trash.pl	(revision 41384)
@@ -0,0 +1,146 @@
+#! /usr/bin/env perl
+
+# TESTRUN can be 1, 2, 3, 4 to get further in the tests
+my $TESTRUN = 2;
+
+use DBI;
+use Carp;
+use Pod::Usage qw( pod2usage );
+
+use strict;
+use warnings FATAL => qw( all );
+use Getopt::Long qw( GetOptions :config auto_help auto_version );
+
+my ($srchost,$trashdir);
+
+GetOptions(
+    'srchost=s' => \$srchost,
+    'trashdir=s' => \$trashdir) || pod2usage(2);
+
+my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser );
+my $conffile = '/data/ippc19.0/home/eugene/rebalance.20200605/nebulous.config';
+
+if (-e $conffile) {
+#   printf STDERR "found config\n";
+    open(IN,$conffile);
+    while(<IN>) {
+        chomp;
+        if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; }
+        if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; }
+        if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; }
+        if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; }
+        if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; }
+    }
+    close(IN);
+    print STDERR  "$dbname $dbhost $dbport $dbuser $dbpass\n";
+}
+
+pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless
+    defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass;
+
+pod2usage( -msg => "Cannot determine trash directory", -exitval => 2) unless defined $trashdir && defined $srchost;
+
+my  $host_check = `hostname`; chomp($host_check);
+if ($host_check ne $srchost) { die "Required to run on $srchost, not on $host_check\n"; }
+
+my $dsn;
+if ($dbport eq "NONE") {
+    $dsn = "DBI:mysql:database=$dbname:host=$dbhost";
+} else {
+    $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport";
+}
+
+my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n";
+
+my @files = <${trashdir}/*>;
+
+for (my $i = 0; $i < @files; $i++) {
+
+    my $filename = $files[$i];
+    
+    my $basefile = $filename;
+    $basefile =~ s/^$trashdir\///;
+    
+    # print "BASEFILE: $basefile\n";
+
+    # the filename should have the following form:
+    # NN/DDDDDDDDDD.WWW:WWW:WWWW:WWW:
+    # NN : 2-hexdigit name of the subsubdir
+    # DDDDD : instance ID (long)
+    # WWW:WWW:WWW : elements of the filename, with : replacing / in the neb storage_object name
+
+    # I want to check two things:
+    # 1) does the file exist at /data/$tgthost.$tgtvol/nebulous/$subdir/$movefile
+    # 2) does the vol_id of the instance_id match our vol_id ?
+
+    my ($ins_id) = $basefile =~ m|^(\d+)\..*|;
+    my $summary = "$filename";
+
+    if ( ! defined ($ins_id)) {
+	print "ERROR : INVALID ID: $summary\n";
+	next;
+    }
+
+    if ( $ins_id =~ m/\D/ ) {
+	print "ERROR : INVALID ID: $summary\n";
+	next;
+    }
+    $summary = "$summary -> $ins_id";
+    
+    # check that this instance exists in DB (if not, need to add it)
+    my $uri_query = "SELECT uri FROM instance WHERE ins_id = ${ins_id}";
+    
+    my $rowref = $dbh->selectall_arrayref( $uri_query );
+    my @array = @$rowref;
+    my $Nrows = @array;
+
+    if ($Nrows == 0) { # We should find this instance
+	print "ERROR : NO DB ENTRY: $summary\n";
+	next;
+    }
+
+    if ($Nrows > 1) { # We should only find one instance
+	print "ERROR : TOO MANY INSTANCE ENTRIES: $summary\n";
+	next;
+    }
+
+    my $valref = $array[0];
+    my @values = @$valref;
+    
+    my $Nvalue = @values;
+
+    if ($Nvalue != 1) {
+	print "ERROR : INVALID DB RESPONSE: $summary\n";
+	next;
+    }
+
+    my $uri_actual = $values[0];
+
+    # strip off the file:// portion:
+    # do we ever have file:/XXX/ ??
+
+    $uri_actual =~ s%file://%%;
+
+    if (-e $uri_actual) { 
+	print "PASSED INSTANCE CHECK: $summary\n";
+    } else {
+	print "ERROR : MISSING INSTANCE : $uri_actual : $ins_id\n";
+    }
+} # END loop over movefiles
+
+# example of using the return value of arrayref:
+
+# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query );
+# my @array = @$rowref;
+# my $Nrows = @array;
+
+# print "NROWS: $Nrows\n";
+
+# foreach my $valref (@array) {
+#     my @values = @$valref;
+#     foreach my $value (@values) {
+#         print "VAL: $value, ";
+#     }
+#     print "\n";
+# }
+
Index: trunk/tools/eam/rebalance/outline.txt
===================================================================
--- trunk/tools/eam/rebalance/outline.txt	(revision 41384)
+++ trunk/tools/eam/rebalance/outline.txt	(revision 41384)
@@ -0,0 +1,32 @@
+
+Notes on the rebalancing script:
+
+* rebalancing is performed for one source host at a time
+* a target number of bytes to remove should be determined from the current neb distribution
+  (goal is uniform fractional fill-factor)
+
+
+Stage 1 : Copy files from source machine
+
+* Ngoal = total number of bytes to remove from the machine
+* for each of the 256 nebulous partitions (nebulous/00 - nebulous/ff), we will select files for a total of Ngoal / 256 bytes
+* generate a list of the files to copy (for each partition)
+* rsync using the input file list to the target
+
+EXAMPLE:
+
+  rebalance.s1.mana /data/ipp070.0/nebulous /data/ipp133.0/rebalance 0.8
+
+  * list of files to move : SRCDIR/NN/movefiles.DATE.txt
+  
+
+Stage 2 : re-insert the files in nebulous
+
+* this is basically identical to the reinsert process from the shuffle work
+
+(do a double-check on the re-inserted files in neb)
+
+Stage 3 : remove the files from the source machine
+
+****
+