Index: trunk/tools/eam/rebalance/duplicates.sh
===================================================================
--- trunk/tools/eam/rebalance/duplicates.sh	(revision 41406)
+++ trunk/tools/eam/rebalance/duplicates.sh	(revision 41406)
@@ -0,0 +1,54 @@
+#!/usr/bin/env mana
+
+# $outdir = dupcount.20200613
+$outdir = dupcount.20200707
+
+macro go
+  dbconnect nebulous ippuser nebulous -p ipp
+
+  mkdir $outdir
+  for i 0 1000
+    echo $i
+    $so_min = $i * 10000000
+    $so_max = $so_min + 10000000
+    dbselect Nins, count(Nins) as Nval from (select so_id, ins_id, count(ins_id) as Nins from storage_object join instance using (so_id) where (so_id >= $so_min) && (so_id < $so_max) group by so_id) as tmp group by Nins
+    sprintf name $outdir/count.%04d.txt $i
+    write $name Nins Nval
+    output $outdir/count.files.txt
+    echo "$name $so_min $so_max"
+    output stdout
+  end
+end
+
+macro check.range
+  if ($0 != 5)
+    echo "USAGE: check.range (min) (max) (Nsub) (outname)"
+    break
+  end
+
+  dbconnect nebulous ippuser nebulous -p ipp
+
+  $so_min_range = $1
+  $so_max_range = $2
+  $outroot = $4
+
+  $del = int(($so_max_range - $so_min_range) / $3)
+  $Nsub = $3 + 1
+
+  mkdir $outdir
+  for i 0 $Nsub
+    $so_min = $so_min_range + $i*$del
+    $so_max = $so_min_range + ($i + 1)*$del
+    dbselect Nins, count(Nins) as Nval from (select so_id, ins_id, count(ins_id) as Nins from storage_object join instance using (so_id) where (so_id >= $so_min) && (so_id < $so_max) group by so_id) as tmp group by Nins
+    sprintf name $outdir/$outroot.%04d.txt $i
+    write $name Nins Nval
+    output $outdir/$outroot.files.txt
+    echo "$name $so_min $so_max"
+    output stdout
+  end
+end
+
+if ($SCRIPT)
+ go
+ exit 0
+end
Index: trunk/tools/eam/rebalance/neb_check_trash.pl
===================================================================
--- trunk/tools/eam/rebalance/neb_check_trash.pl	(revision 41405)
+++ trunk/tools/eam/rebalance/neb_check_trash.pl	(revision 41406)
@@ -12,9 +12,10 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version );
 
-my ($srchost,$trashdir);
+my ($srchost,$trashdir,$savedir);
 
 GetOptions(
-    'srchost=s' => \$srchost,
-    'trashdir=s' => \$trashdir) || pod2usage(2);
+    'srchost=s'  => \$srchost,
+    'trashdir=s' => \$trashdir,
+    'savedir=s'  => \$savedir) || pod2usage(2);
 
 my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser );
@@ -39,5 +40,5 @@
     defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass;
 
-pod2usage( -msg => "Cannot determine trash directory", -exitval => 2) unless defined $trashdir && defined $srchost;
+pod2usage( -msg => "Cannot determine srchost, trashdir, or savedir", -exitval => 2) unless defined $trashdir && defined $savedir && defined $srchost;
 
 my  $host_check = `hostname`; chomp($host_check);
@@ -52,4 +53,6 @@
 
 my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n";
+
+unless (mkdir $savedir) { die "failed to make savedir\n"; }
 
 my @files = <${trashdir}/*>;
@@ -96,5 +99,6 @@
 
     if ($Nrows == 0) { # We should find this instance
-	print "ERROR : NO DB ENTRY: $summary\n";
+	print "ERROR : NO DB ENTRY: (saving) $summary\n";
+	system ("mv $filename $savedir/");
 	next;
     }
Index: trunk/tools/eam/rebalance/neb_link_repair.pl
===================================================================
--- trunk/tools/eam/rebalance/neb_link_repair.pl	(revision 41406)
+++ trunk/tools/eam/rebalance/neb_link_repair.pl	(revision 41406)
@@ -0,0 +1,46 @@
+#!/usr/bin/env perl
+
+if (@ARGV < 1) { die "USAGE: neb_link_repair.pl (filename)\n"; }
+
+my $file = $ARGV[0];
+
+# validate the file (exists and is a link)
+if (! -l $file) { die "file $file is not a link\n"; }
+
+# if the linked file exists, do not relink
+if (  -e $file) { die "file $file is linked to a valid file\n"; }
+
+# get the value of the link:
+my $linkvalue = readlink $file;
+
+# print "file: $file\n";
+# print "link: $linkvalue\n";
+
+# convert the filename 
+my (@neb_path) = split /\//, $linkvalue;
+my $neb_key = $neb_path[-1];
+# Remove ins_id
+$neb_key =~ s/^\d+\.//;
+# Convert colons to slashes.
+$neb_key =~ s%:%/%g;
+$neb_key =~ s%.fz$%%;
+
+# print "nebk: $neb_key\n";
+
+$neb_file = `neb-ls -p $neb_key`;
+chomp $neb_file;
+if ("$neb_file" eq "") { die "nebulous entry is missing\n";}
+
+$savelink = $file . ".broken";
+if (-e $savelink) { die "save link $savelink already exists, remove first\n"; }
+
+system ("mv $file $savelink");
+# print "mv $file $savelink\n";
+
+my $linkstatus = symlink $neb_file,$file;
+# print "link $neb_file to $file\n";
+
+if (! $linkstatus) { die "error relinking $neb_file to $file\n" }
+
+exit 0;
+
Index: trunk/tools/eam/rebalance/rebalance.mana
===================================================================
--- trunk/tools/eam/rebalance/rebalance.mana	(revision 41405)
+++ trunk/tools/eam/rebalance/rebalance.mana	(revision 41406)
@@ -158,5 +158,5 @@
     sprintf movelist %s/rebalance/movefiles.%s.txt $srcdir $VERSION
     sprintf movelog  %s/rebalance/movefiles.%s.log $srcdir $VERSION
-    sprintf subdirset "%s/0?" $srcdir
+    sprintf subdirset "%s/??" $srcdir
     exec find $subdirset -size +$BIGSIZE > $filelist
     list files -x "/bin/ls -l `cat $filelist` | sort -k 5nr"
@@ -228,4 +228,39 @@
 end
 
+macro run.rsync.check
+  if ($0 != 7)
+    echo "USAGE: run.rsync.check (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) (OUTVER)"
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    break
+  end
+
+  local srchost srcvol tgthost tgtvol
+
+  $srchost = $1
+  $srcvol = $2
+  $tgthost = $3
+  $tgtvol = $4
+  $VERSION = $5
+  $OUTVER  = $6
+
+  # generate the full paths:
+  $source = /data/$srchost.$srcvol/nebulous
+  $target = /data/$tgthost.$tgtvol/rebalance/$srchost.$srcvol/nebulous
+
+  for i 0 $NSUBDIR
+    sprintf srcdir "%s/%02x" $source $i
+    sprintf tgtdir "%s/%02x" $target $i
+    # echo $srcdir
+
+    # generate initial file list (all big files)
+    sprintf movelist %s/rebalance/movefiles.%s.txt $srcdir $VERSION
+    sprintf movelog  %s/rebalance/movefiles.%s.log $srcdir $OUTVER
+
+    echo "rsyncing $srcdir/ to $tgtdir/ : $movelog"
+    exec rsync -auv --files-from=$movelist $srcdir/ $tgtdir/ >& $movelog
+  end
+end
+
 # names are defined for a single src & tgt pair
 macro run.insert
@@ -287,7 +322,7 @@
 
 # names are defined for a single src & tgt pair
-macro run.check
+macro run.insert.check
   if ($0 != 6)
-    echo "USAGE: run.check (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+    echo "USAGE: run.insert.check (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
     echo "srchost, tgthost : e.g., ipp070"
     echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
@@ -384,7 +419,7 @@
 end
 
-macro run.check.trash
+macro run.trash.check
   if ($0 != 6)
-    echo "USAGE: run.check.trash (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+    echo "USAGE: run.trash.check (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
     echo "srchost, tgthost : e.g., ipp070"
     echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
@@ -407,8 +442,9 @@
   for i 0 $NSUBDIR
     sprintf trashdir "/data/%s.%s/nebulous/%02x/trash" $srchost $srcvol $i
+    sprintf savedir  "/data/%s.%s/nebulous/%02x/save" $srchost $srcvol $i
     sprintf trashlog "/data/%s.%s/nebulous/%02x/rebalance/check.trash.%s.log" $srchost $srcvol $i $VERSION
 
-     echo ./neb_check_trash.pl --srchost $srchost --trashdir $trashdir > $trashlog
-    $EXEC ./neb_check_trash.pl --srchost $srchost --trashdir $trashdir > $trashlog
+     echo ./neb_check_trash.pl --srchost $srchost --trashdir $trashdir --savedir $savedir > $trashlog
+    $EXEC ./neb_check_trash.pl --srchost $srchost --trashdir $trashdir --savedir $savedir > $trashlog
   end
 
@@ -455,13 +491,16 @@
 
     # list of files which have been moved & checked
-    sprintf movelist /data/%s.%s/nebulous/%02x/rebalance/movefiles.%s.txt $srchost $srcvol $i $VERSION
-
-    list files -x "cat $movelist"
+#   sprintf movelist /data/%s.%s/nebulous/%02x/rebalance/movefiles.%s.txt $srchost $srcvol $i $VERSION
+
+    list files -glob $trashdir/*
     for j 0 $files:n
-      basename $files:$j -var myfile
-
+      file $files:$j -can-write canWrite
+      if (not($canWrite))
+        echo "SKIP write-protected file : $files:$j"
+        next
+      end    
       # remove the file from the trash directory
-      echo rm $trashdir/$myfile
-      exec rm $trashdir/$myfile
+      echo rm $files:$j
+      exec rm $files:$j
     end
   end
@@ -469,8 +508,8 @@
 
 if ($SCRIPT) 
-  if ($argv:n < 6) 
+  if ($argv:n < 1) 
     echo "USAGE: rebalance.mana (MODE) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) [options]"
     echo ""
-    echo "MODE: find.test, find, rsync, insert, check, trash, check.trash, empty.trash"
+    echo "MODE: find.test, find, rsync, rsync.check, insert, insert.check, trash, trash.check, empty.trash"
     echo ""
     echo "srchost, tgthost : e.g., ipp070"
@@ -513,4 +552,14 @@
   end
 
+  #### rsync.check ####
+  if ("$argv:0" == "rsync.check")
+    if ($argv:n != 7)
+      echo "USAGE: rebalance.mana (rsync.check) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) (OUTVER)"
+      break
+    end
+    run.rsync.check $argv:1 $argv:2 $argv:3 $argv:4 $argv:5 $argv:6
+    exit 0
+  end
+
   #### insert ####
   if ("$argv:0" == "insert")
@@ -523,11 +572,11 @@
   end
 
-  #### check ####
-  if ("$argv:0" == "check")
+  #### insert.check ####
+  if ("$argv:0" == "insert.check")
     if ($argv:n != 6)
-      echo "USAGE: rebalance.mana (check) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
-      break
-    end
-    run.check $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
+      echo "USAGE: rebalance.mana (insert.check) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+      break
+    end
+    run.insert.check $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
     exit 0
   end
@@ -543,11 +592,11 @@
   end
 
-  #### check.trash ####
-  if ("$argv:0" == "check.trash")
+  #### trash.check ####
+  if ("$argv:0" == "trash.check")
     if ($argv:n != 6)
-      echo "USAGE: rebalance.mana (check.trash) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
-      break
-    end
-    run.check.trash $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
+      echo "USAGE: rebalance.mana (trash.check) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+      break
+    end
+    run.trash.check $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
     exit 0
   end
@@ -556,5 +605,5 @@
   if ("$argv:0" == "empty.trash")
     if ($argv:n != 6)
-      echo "USAGE: rebalance.mana (check.trash) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+      echo "USAGE: rebalance.mana (empty.trash) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
       break
     end
@@ -563,4 +612,5 @@
   end
 
-  exit 0
-end
+  echo "ERROR: unknown mode $argv:0"
+  exit 1
+end