Index: trunk/Nebulous-Server/bin/neb-admin
===================================================================
--- trunk/Nebulous-Server/bin/neb-admin	(revision 29071)
+++ trunk/Nebulous-Server/bin/neb-admin	(revision 29425)
@@ -27,4 +27,7 @@
     $limit,
     $pending,
+    $balance,
+    $balance_N_sources,
+    $balance_M_destinations,
     $removal,
     $verbose,
@@ -42,5 +45,8 @@
     'pass|p=s'              => \$dbpass,
     'pendingreplicate|r'    => \$pending,
-    'pendingremoval'        => \$removal,
+    'pendingbalance|b'      => \$balance,
+    'balancesources|N=s'      => \$balance_N_sources,
+    'balancedestinations|M=s' => \$balance_M_destinations,
+#    'pendingremoval'        => \$removal,
     'so_id_start=i'         => \$so_id_start,
     'so_id_range=i'         => \$so_id_range,
@@ -54,8 +60,10 @@
 pod2usage( -msg => "--limit is meaningless without --pendingreplicate",
         -exitval => 2 )
-    if defined $limit and not defined $pending;
+    if defined $limit and not (defined $pending or defined $balance);
 pod2usage( -msg => "no operation specified", -exitval => 2 )
-    unless defined $pending or defined $removal;
-
+    unless defined $pending or defined $balance;
+pod2usage( -msg => "--pendingbalance needs --balancesources and --balancedestinations options",
+	   -exitval => 2)
+    if defined $balance and not (defined $balance_N_sources or defined $balance_M_destinations);
 # set default limit to 5
 $limit ||= 5;
@@ -84,4 +92,6 @@
 } elsif ($removal) {
     removal();
+} elsif ($balance) {
+    balance();
 } else {
     die "THIS SHOULD NOT HAPPEN";
@@ -137,10 +147,9 @@
             AND so.so_id <  $so_id_end
         GROUP BY so_id
-        HAVING available_instances < instances OR instances < copies
+        HAVING available_instances != instances OR instances != copies
         LIMIT $limit"
     );
+
     $query->execute;
-
-#    $dbh->do("DROP TABLE IF EXISTS mymountedvol");
 
     my @rows;
@@ -168,5 +177,5 @@
         my $vol_name = $obj->{volume_name};
         my $vol_host = $obj->{volume_host};
-        my $available = $obj->{available_instances} || 0;
+	my $available = $obj->{available_instances} || 0;
         my $need_recovery = $obj->{need_recovery};
         my $recoverable = $obj->{recoverable};
@@ -189,16 +198,219 @@
 
         my $need = $copies - $available;
-        $need = 0 if $need < 0;
-
+	my $cmd = 'neb-stat';
+#        $need = 0 if $need < 0;
         next unless $need;
+
+	if ($need > 0) {
+	    $cmd = 'neb-replicate';
+	}
+	elsif ($need < 0) {
+	    $cmd = 'neb-cull';
+	}
 
         my %md = (
             key         => $key,
+	    command     => $cmd,
             need_copies => $need,
             volume_name => $vol_name,
             volume_host => $vol_host,
+
+#	    has         => $has,
+	    available   => $available,
+	    xattr_copies      => $copies,
+#	    recoverable => $recoverable,
+#	    need_recovery => $need_recovery,
+	    
         );
-
+#	print "$key $cmd $need $vol_name $vol_host\n";
         print_metadata("replicatePending", \%md);
+        print "\n";
+    }
+
+    # use a different exit status if we hit the limit (likely more files pending)
+    if ($Npending == $limit) {
+        exit 1;
+    } 
+    exit 0;
+}
+
+sub balance
+{
+# so_id, ext_id, instances, available_instances, need_recovery, recoverable
+# XXX don't remove the temp table code -- testing w/o it
+#    $dbh->do("CREATE TEMPORARY TABLE mymountedvol LIKE mountedvol");
+#    $dbh->do("INSERT INTO mymountedvol SELECT * FROM mountedvol");
+
+    if (not defined $so_id_start) { $so_id_start = 0; }
+    if (not defined $so_id_range) { $so_id_range = 100000; }
+    my $so_id_end = $so_id_start + $so_id_range;
+
+    # XXX check if so_id_start is beyond MAX(so_id): if so, exit with exit status 10
+    { 
+        my $query = $dbh->prepare("SELECT MAX(so_id) from storage_object");
+        $query->execute;
+        my $answer = $query->fetchrow_arrayref;
+        my $max_so_id = $$answer[0];
+        $query->finish;
+
+        if ($so_id_start > $max_so_id) { 
+            print STDERR "at end of so_id range, reset please\n";
+            exit 10;
+        }
+    }
+
+    # Calculate the average disk usage
+    my $average = 0;
+    my $avg_query = $dbh->prepare(
+	"select sum(used)/sum(total) AS average from mountedvol WHERE available = 1 AND allocate = 1"
+	);
+    $avg_query->execute();
+    while (my $row = $avg_query->fetchrow_hashref) {
+	$average = $row->{average};
+    }
+    $avg_query->finish;
+    print STDERR "average: $average\n";
+    if ($average == 0) {
+	exit 1;
+    }
+    # I truly am sorry for this statement.
+    my $query = $dbh->prepare(
+	"
+SELECT * FROM (
+ -- U Randomize the list of all possible source/destination matches,then group by so_id to select one at random
+ SELECT T.*,volume.cab_id AS source_cab_id,volume.name AS source_name,volume.host AS source_host,
+        destination.vol_id AS destination_vol_id,destination.cab_id AS destination_cab_id,
+        destination.name AS destination_name,destination.host AS destination_host FROM (
+  -- V Ensure that the copy we found on the source volume is the second copy instance
+  SELECT K.vol_id AS here,K.ins_id,K.so_id,ext_id,value AS user_copies,MAXins_id,instance.vol_id AS there FROM (
+   -- W Determine what the second copy instance is
+   SELECT V.*,MAX(instance.ins_id) AS MAXins_id FROM (
+    -- X Determine which hosts have instances of an object
+    SELECT vol_id,ins_id,so_id,ext_id,value FROM 
+    storage_object JOIN storage_object_xattr USING(so_id) JOIN instance USING(so_id)
+    WHERE vol_id IN (
+     -- Y Extra select for debugging purposes
+     SELECT vol_id FROM (
+      -- Z Ran hosts by R (need to be a source) and return top N
+      SELECT mountedvol.vol_id,mountedvol.name,mountedvol.available,mountedvol.allocate,total,used,
+             (used / total) AS D,total * ((used / total) - $average ) AS R,  -- average
+             cab_id FROM
+      mountedvol JOIN volume USING(vol_id)
+      WHERE mountedvol.available = 1 AND mountedvol.allocate = 1
+      ORDER BY R DESC LIMIT $balance_N_sources                                            -- N
+      ) AS ranked_sources
+      -- Z End
+     )
+     -- Y End
+    AND name = 'user.copies' AND value >= 2
+    AND so_id >= $so_id_start                                  -- so_id_start
+    AND so_id <  $so_id_end                                    -- so_id_end
+    LIMIT $limit                                               -- limit
+   ) AS V -- volumes that host a copy
+   -- X End
+  LEFT OUTER JOIN instance USING(so_id) GROUP BY so_id
+  ) AS K -- copies that are the second copy
+  -- W End
+ JOIN instance ON MAXins_id = instance.ins_id GROUP BY so_id
+ ) AS T -- matched copies that I know the second copy is on my source volume
+ -- V End
+ JOIN volume on T.here = volume.vol_id RIGHT JOIN (
+  -- Y2 Chose a random destination from the list
+  SELECT vol_id,cab_id,name,host FROM (
+   -- Z2 Reverse rank hosts by R and return top M
+   SELECT mountedvol.vol_id,mountedvol.name,mountedvol.host,mountedvol.available,mountedvol.allocate,total,used,
+          (used / total) AS D,total * ((used / total) - $average ) AS R,  -- average
+          cab_id FROM
+   mountedvol JOIN volume USING(vol_id)
+   WHERE mountedvol.available = 1 AND mountedvol.allocate = 1
+   ORDER BY R ASC LIMIT $balance_M_destinations                                               -- M
+  ) AS ranked_destinations
+  -- Z2 End
+ ORDER BY RAND()
+ ) AS destination
+ -- Y2 End
+ ON destination.cab_id != volume.cab_id
+ WHERE here = there
+ ORDER BY RAND()
+) AS results
+-- U End
+GROUP BY so_id
+"    );
+
+    $query->execute();
+		    
+		    
+
+    my @rows;
+    while (my $row = $query->fetchrow_hashref) {
+        push @rows, $row;        
+    }
+    $query->finish;
+
+    print STDERR "No rows found\n" unless scalar @rows;
+    exit unless scalar @rows;
+    
+    # compare number of responses to limit below
+    my $Npending = @rows;
+
+    print "balancePending MULTI\n\n";
+
+    foreach my $obj (@rows) {
+        if (defined $verbose) {
+            require Data::Dumper;
+            print Data::Dumper::Dumper($obj);
+        }
+	
+
+	my $here = $obj->{here};
+	my $there = $obj->{there};
+	my $ins_id = $obj->{ins_id};
+	my $max_ins_id = $obj->{MAXins_id};
+	
+	my $so_id = $obj->{so_id};
+	my $key   = $obj->{ext_id};
+	my $copies = $obj->{user_copies};
+
+	my $source_vol_id = $obj->{here};
+	my $source_cab_id = $obj->{source_cab_id};
+	my $source_name   = $obj->{source_name};
+	my $source_host   = $obj->{source_host};
+
+	my $destination_vol_id = $obj->{destination_vol_id};
+	my $destination_cab_id = $obj->{destination_cab_id};
+	my $destination_name   = $obj->{destination_name};
+	my $destination_host   = $obj->{destination_host};
+	
+	unless ($here == $there) { next; }
+	unless ($ins_id == $max_ins_id) { next; }
+	unless ($source_cab_id != $destination_cab_id) { next; }
+	
+        my %md = (
+	    key           => $key,
+	    copies        => $copies,
+	    so_id         => $so_id,
+	    source_vol_id => $source_vol_id,
+	    source_cab_id => $source_cab_id,
+	    source_name   => $source_name,
+	    source_host   => $source_host,
+	    destination_vol_id => $destination_vol_id,
+	    destination_cab_id => $destination_cab_id,
+	    destination_name   => $destination_name,
+	    destination_host   => $destination_host,
+#             key         => $key,
+# 	    command     => $cmd,
+#             need_copies => $need,
+#             volume_name => $vol_name,
+#             volume_host => $vol_host,
+
+# #	    has         => $has,
+# 	    available   => $available,
+# 	    xattr_copies      => $copies,
+# #	    recoverable => $recoverable,
+# #	    need_recovery => $need_recovery,
+	    
+        );
+#	print "$key $cmd $need $vol_name $vol_host\n";
+        print_metadata("balancePending", \%md);
         print "\n";
     }