Index: trunk/tools/eam/rebalance/launch.insert.check.errors.sh
===================================================================
--- trunk/tools/eam/rebalance/launch.insert.check.errors.sh	(revision 41509)
+++ trunk/tools/eam/rebalance/launch.insert.check.errors.sh	(revision 41509)
@@ -0,0 +1,31 @@
+#!/bin/csh 
+
+set Nhost = `hostname | sed s/ipp//`
+
+set ver = "v0"
+set verdate = "20201209"
+
+
+set target = NONE
+if (($Nhost > 101) && ($Nhost <= 111)) then 
+  set target = ipp145
+endif
+if ($Nhost > 111) then 
+  set target = ipp146
+endif
+
+if (($Nhost == 100) || ($Nhost == 101)) then
+  set target = ipp140
+  set v = 0 ; set h = `hostname` ; set t = $target; grep ERROR /data/$h.$v/nebulous/??/rebalance/movefiles.$verdate.chk >& $h.$v/errors.insert.check.$verdate.$ver &
+  set target = ipp133
+  set v = 1 ; set h = `hostname` ; set t = $target; grep ERROR /data/$h.$v/nebulous/??/rebalance/movefiles.$verdate.chk >& $h.$v/errors.insert.check.$verdate.$ver &
+  exit 0
+endif
+
+if ("$target" == "NONE") then
+  echo "need target"
+  exit 2
+endif
+
+set v = 0 ; set h = `hostname` ; set t = $target; grep ERROR /data/$h.$v/nebulous/??/rebalance/movefiles.$verdate.chk >& $h.$v/errors.insert.check.$verdate.$ver &
+set v = 1 ; set h = `hostname` ; set t = $target; grep ERROR /data/$h.$v/nebulous/??/rebalance/movefiles.$verdate.chk >& $h.$v/errors.insert.check.$verdate.$ver &
Index: trunk/tools/eam/rebalance/launch.insert.errors.sh
===================================================================
--- trunk/tools/eam/rebalance/launch.insert.errors.sh	(revision 41509)
+++ trunk/tools/eam/rebalance/launch.insert.errors.sh	(revision 41509)
@@ -0,0 +1,31 @@
+#!/bin/csh 
+
+set Nhost = `hostname | sed s/ipp//`
+
+set ver = "v0"
+set verdate = "20201209"
+
+
+set target = NONE
+if (($Nhost > 101) && ($Nhost <= 111)) then 
+  set target = ipp145
+endif
+if ($Nhost > 111) then 
+  set target = ipp146
+endif
+
+if (($Nhost == 100) || ($Nhost == 101)) then
+  set target = ipp140
+  set v = 0 ; set h = `hostname` ; set t = $target; grep ERROR /data/$h.$v/nebulous/??/rebalance/insert.$verdate.log >& $h.$v/errors.insert.$verdate.$ver &
+  set target = ipp133
+  set v = 1 ; set h = `hostname` ; set t = $target; grep ERROR /data/$h.$v/nebulous/??/rebalance/insert.$verdate.log >& $h.$v/errors.insert.$verdate.$ver &
+  exit 0
+endif
+
+if ("$target" == "NONE") then
+  echo "need target"
+  exit 2
+endif
+
+set v = 0 ; set h = `hostname` ; set t = $target; grep ERROR /data/$h.$v/nebulous/??/rebalance/insert.$verdate.log >& $h.$v/errors.insert.$verdate.$ver &
+set v = 1 ; set h = `hostname` ; set t = $target; grep ERROR /data/$h.$v/nebulous/??/rebalance/insert.$verdate.log >& $h.$v/errors.insert.$verdate.$ver &
Index: trunk/tools/eam/rebalance/launch.rsync.check.sh
===================================================================
--- trunk/tools/eam/rebalance/launch.rsync.check.sh	(revision 41509)
+++ trunk/tools/eam/rebalance/launch.rsync.check.sh	(revision 41509)
@@ -0,0 +1,31 @@
+#!/bin/csh 
+
+set Nhost = `hostname | sed s/ipp//`
+
+set ver = "v1"
+set verdate = "20201209"
+
+
+set target = NONE
+if (($Nhost > 101) && ($Nhost <= 111)) then 
+  set target = ipp145
+endif
+if ($Nhost > 111) then 
+  set target = ipp146
+endif
+
+if (($Nhost == 100) || ($Nhost == 101)) then
+  set target = ipp140
+  set v = 0 ; set h = `hostname` ; set t = $target; ./rebalance.mana rsync.check $h $v $t $v $verdate $verdate.chk > & $h.$v/log.rsync.check.$verdate.$ver &
+  set target = ipp133
+  set v = 1 ; set h = `hostname` ; set t = $target; ./rebalance.mana rsync.check $h $v $t $v $verdate $verdate.chk > & $h.$v/log.rsync.check.$verdate.$ver &
+  exit 0
+endif
+
+if ("$target" == "NONE") then
+  echo "need target"
+  exit 2
+endif
+
+set v = 0 ; set h = `hostname` ; set t = $target; ./rebalance.mana rsync.check $h $v $t $v $verdate $verdate.chk > & $h.$v/log.rsync.check.$verdate.$ver &
+set v = 1 ; set h = `hostname` ; set t = $target; ./rebalance.mana rsync.check $h $v $t $v $verdate $verdate.chk > & $h.$v/log.rsync.check.$verdate.$ver &
Index: trunk/tools/eam/rebalance/launch.rsync.sh
===================================================================
--- trunk/tools/eam/rebalance/launch.rsync.sh	(revision 41509)
+++ trunk/tools/eam/rebalance/launch.rsync.sh	(revision 41509)
@@ -0,0 +1,22 @@
+#!/bin/csh 
+
+set Nhost = `hostname | sed s/ipp//`
+
+set target = NONE
+if (($Nhost > 101) && ($Nhost <= 111)) then 
+  set target = ipp145
+endif
+if ($Nhost > 111) then 
+  set target = ipp146
+endif
+
+if ("$target" == "NONE") then
+  echo "need target"
+  exit 2
+endif
+
+#set v = 0 ; set h = `hostname` ; set t = $target ; mkdir $h.$v; ./rebalance.mana find $h $v $t $v 20201209 0.9 > & $h.$v/log.find.20201209.v1 &
+#set v = 1 ; set h = `hostname` ; set t = $target ; mkdir $h.$v; ./rebalance.mana find $h $v $t $v 20201209 0.9 > & $h.$v/log.find.20201209.v1 &
+
+set v = 0 ; set h = `hostname` ; set t = $target; ./rebalance.mana rsync $h $v $t $v 20201209 > & $h.$v/log.rsync.20201209.v0 &
+set v = 1 ; set h = `hostname` ; set t = $target; ./rebalance.mana rsync $h $v $t $v 20201209 > & $h.$v/log.rsync.20201209.v0 &
Index: trunk/tools/eam/rebalance/neb_check_location.pl
===================================================================
--- trunk/tools/eam/rebalance/neb_check_location.pl	(revision 41509)
+++ trunk/tools/eam/rebalance/neb_check_location.pl	(revision 41509)
@@ -0,0 +1,244 @@
+#! /usr/bin/env perl
+
+# TESTRUN can be 1, 2, 3, 4 to get further in the tests
+my $TESTRUN = 2;
+
+use DBI;
+use Carp;
+use Pod::Usage qw( pod2usage );
+
+use strict;
+use warnings FATAL => qw( all );
+use Getopt::Long qw( GetOptions :config auto_help auto_version );
+
+my ($tgthost,$tgtvol,$subdir,$movelist);
+
+GetOptions(
+    'tgthost=s' => \$tgthost,
+    'tgtvol=s'  => \$tgtvol,
+    'subdir=s'  => \$subdir,
+    'list=s'    => \$movelist) || pod2usage(2);
+
+my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser );
+my $conffile = '/data/ippc19.0/home/eugene/rebalance.20200605/nebulous.config';
+
+if (-e $conffile) {
+#   printf STDERR "found config\n";
+    open(IN,$conffile);
+    while(<IN>) {
+        chomp;
+        if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; }
+        if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; }
+        if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; }
+        if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; }
+        if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; }
+    }
+    close(IN);
+    print STDERR  "$dbname $dbhost $dbport $dbuser $dbpass\n";
+}
+
+pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless
+    defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass;
+
+pod2usage( -msg => "Cannot determine target machine and volume", -exitval => 2) unless
+    defined $tgthost && defined $tgtvol;
+
+pod2usage( -msg => "Cannot determine subdir or list of files which have been moved", -exitval => 2) unless
+    defined $movelist && defined $subdir;
+
+my $host_check = `hostname`; chomp($host_check);
+
+if ($host_check ne $tgthost) { die "Required to run on $tgthost, not on $host_check\n"; }
+
+my $time = time();
+
+my $dsn;
+if ($dbport eq "NONE") {
+    $dsn = "DBI:mysql:database=$dbname:host=$dbhost";
+} else {
+    $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport";
+}
+
+my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n";
+
+# example:
+# tgthost : ipp130
+# tgtvol  : 0 
+
+# Get the vol_id of the (local) destination.
+
+my $tgt_volume = "${tgthost}.${tgtvol}";
+my $tgt_subdir = "/export/${tgthost}.${tgtvol}/nebulous/${subdir}";
+
+### XX my $sql_get_volume_id = "SELECT vol_id, name, host, path FROM volume WHERE name = ?";
+### XX my $sql_update_instance = "UPDATE instance SET vol_id = ?, uri = ? WHERE ins_id = ?";
+### XX 
+### XX my $query = $dbh->prepare( $sql_get_volume_id );
+### XX 
+### XX $query->execute( $tgt_volume );
+### XX my ($tgt_vol_id, $tgt_vol_name, $tgt_vol_host, $tgt_vol_path) = $query->fetchrow_array;
+### XX $query->finish;
+### XX 
+### XX print "LOG INIT $tgthost $tgtvol $tgt_vol_id\n";
+
+# load the list of files from movelist
+
+open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n";
+my @movefiles = <MOVELIST>;
+close (MOVELIST);
+
+my $otime = $time;
+$time = time();
+my $dtime = $time - $otime;
+
+for (my $i = 0; $i < @movefiles; $i++) {
+
+    my $movefile = $movefiles[$i];
+    chomp $movefile;
+    
+    # the filename should have the following form:
+    # NN/DDDDDDDDDD.WWW:WWW:WWWW:WWW:
+    # NN : 2-hexdigit name of the subsubdir
+    # DDDDD : instance ID (long)
+    # WWW:WWW:WWW : elements of the filename, with : replacing / in the neb storage_object name
+
+    # I want to check two things:
+    # 1) does the file exist at /data/$tgthost.$tgtvol/nebulous/$subdir/$movefile
+    # 2) does the vol_id of the instance_id match our vol_id ?
+
+    unless (-e "$tgt_subdir/$movefile") {
+	print "ERROR : MISSING FILE: $tgt_subdir/$movefile\n";
+	next;
+    }
+
+    # print "movefile: $movefile\n";
+
+    my ($ins_id) = $movefile =~ m|^[0-9a-fA-F][0-9a-fA-F]/(\d+)\..*|;
+    my $summary = "$tgt_subdir/$movefile";
+
+    if ( ! defined ($ins_id)) {
+	print "ERROR : INVALID ID: $summary\n";
+	next;
+    }
+
+    if ( $ins_id =~ m/\D/ ) {
+	print "ERROR : INVALID ID: $summary\n";
+	next;
+    }
+    $summary = "$summary : $ins_id";
+    
+    # get so_id for this ins_id:
+    my $so_id = 0;
+    if (1) {
+	my $so_id_query = "SELECT so_id FROM instance WHERE ins_id = ${ins_id}";
+	
+	my $rowref = $dbh->selectall_arrayref( $so_id_query );
+	my @array = @$rowref;
+	my $Nrows = @array;
+
+	if ($Nrows == 0) { # We should find this instance
+	    print "ERROR : NO DB ENTRY: $summary\n";
+	    next;
+	}
+
+	if ($Nrows > 1) { # We should only find one instance
+	    print "ERROR : TOO MANY SO_ID ENTRIES: $summary\n";
+	    next;
+	}
+
+	my $valref = $array[0];
+	my @values = @$valref;
+	
+	my $Nvalue = @values;
+
+	if ($Nvalue != 1) {
+	    print "ERROR : INVALID DB RESPONSE: $summary\n";
+	    next;
+	}
+	$so_id = $values[0];
+    }
+
+    $summary = "$summary : $so_id";
+
+    # get user_copies for this storage_object
+    my $Ncopies = 0;
+    my $ext_id = "";
+    if (1) {
+	my $ncopies_query = "SELECT ext_id, value FROM storage_object JOIN storage_object_xattr USING (so_id) WHERE (so_id = ${so_id}) and (name = 'user.copies')";
+	
+	my $rowref = $dbh->selectall_arrayref( $ncopies_query );
+	my @array = @$rowref;
+	my $Nrows = @array;
+
+	if ($Nrows == 0) { # user.copies is not set, skip
+	    print "SKIP: NO USER.COPIES ENTRY: $summary\n";
+	    next;
+	}
+
+	if ($Nrows > 1) { # We should only find one instance
+	    print "ERROR : TOO MANY XATTR ENTRIES: $summary : $Nrows\n";
+	    next;
+	}
+
+	my $valref = $array[0];
+	my @values = @$valref;
+	
+	my $Nvalue = @values;
+
+	if ($Nvalue != 2) {
+	    print "ERROR : INVALID DB XATTR RESPONSE: $summary\n";
+	    next;
+	}
+	$ext_id = $values[0];
+	$Ncopies = $values[1];
+    }
+
+    $summary = "$summary : $ext_id";
+
+    if ($Ncopies < 2) {
+	print "SKIP: FEWER THAN 2 USER.COPIES REQUESTED: $summary : $Ncopies\n";
+	next;
+    }
+
+    # check for backup instances
+    if (1) {
+	my $ncopies_query = "SELECT ins_id, vol_id FROM instance JOIN volume using (vol_id) WHERE (so_id = ${so_id}) and (xattr = 1)";
+	
+	my $rowref = $dbh->selectall_arrayref( $ncopies_query );
+	my @array = @$rowref;
+	my $Nrows = @array;
+
+	if ($Nrows == 0) { # We should find this instance
+	    print "REPL: NO BACKUP COPY: $summary\n";
+	    next;
+	}
+
+	## my $valref = $array[0];
+	## my @values = @$valref;
+	## 
+	## my $Nvalue = @values;
+	## 
+	## if ($Nvalue != 1) {
+	##     print "ERROR : INVALID DB XATTR RESPONSE: $summary\n";
+	##     next;
+	## }
+    }
+
+} # END loop over movefiles
+
+# example of using the return value of arrayref:
+
+# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query );
+# my @array = @$rowref;
+# my $Nrows = @array;
+
+# print "NROWS: $Nrows\n";
+
+# foreach my $valref (@array) {
+#     my @values = @$valref;
+#     foreach my $value (@values) {
+#         print "VAL: $value, ";
+#     }
+#     print "\n";
+# }
+
Index: trunk/tools/eam/rebalance/neb_fix_location.pl
===================================================================
--- trunk/tools/eam/rebalance/neb_fix_location.pl	(revision 41509)
+++ trunk/tools/eam/rebalance/neb_fix_location.pl	(revision 41509)
@@ -0,0 +1,198 @@
+#! /usr/bin/env perl
+
+# TESTRUN can be 1, 2, 3, 4 to get further in the tests
+my $TESTRUN = 2;
+
+use DBI;
+use Carp;
+use Pod::Usage qw( pod2usage );
+
+use strict;
+use warnings FATAL => qw( all );
+use Getopt::Long qw( GetOptions :config auto_help auto_version );
+
+my ($movelist);
+
+GetOptions( 'list=s'    => \$movelist) || pod2usage(2);
+
+my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser );
+my $conffile = '/data/ippc19.0/home/eugene/rebalance.20200605/nebulous.config';
+
+if (-e $conffile) {
+#   printf STDERR "found config\n";
+    open(IN,$conffile);
+    while(<IN>) {
+        chomp;
+        if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; }
+        if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; }
+        if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; }
+        if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; }
+        if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; }
+    }
+    close(IN);
+    print STDERR  "$dbname $dbhost $dbport $dbuser $dbpass\n";
+}
+
+pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless
+    defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass;
+
+pod2usage( -msg => "Cannot determine list of files which have been moved", -exitval => 2) unless
+    defined $movelist;
+
+my $time = time();
+
+my $dsn;
+if ($dbport eq "NONE") {
+    $dsn = "DBI:mysql:database=$dbname:host=$dbhost";
+} else {
+    $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport";
+}
+
+my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n";
+
+# load the list of files from movelist
+
+open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n";
+my @movefiles = <MOVELIST>;
+close (MOVELIST);
+
+my $otime = $time;
+$time = time();
+my $dtime = $time - $otime;
+
+for (my $i = 0; $i < @movefiles; $i++) {
+
+    my $movefile = $movefiles[$i];
+    chomp $movefile;
+    
+    my @word = split (" ", $movefile);
+
+    my $so_id;
+    my $ext_id;
+    if (0) { 
+	# IF we are reading the output of neb_check_location.pl
+	# each line contains STATE: Message
+	# only operate on the files with STATE = REPL
+
+	unless ($word[0] eq "REPL:") { next; }
+
+	$so_id  = $word[8];
+	$ext_id = $word[10];
+    } else {
+	# IF we are reading the output of neb.location.result.sh, 
+	# only REPL files are included and each line contains
+	# so_id ext_id
+	$so_id  = $word[0];
+	$ext_id = $word[1];
+    }
+
+    if ( ! defined ($so_id)) {
+	print STDERR "ERROR : INVALID SO_ID: $movefile\n";
+	next;
+    }
+
+    if ( ! defined ($ext_id)) {
+	print STDERR "ERROR : MISSING EXT_ID: $movefile\n";
+	next;
+    }
+
+    my $summary = "$ext_id $so_id";
+
+    # get user_copies for this storage_object
+    my $Nmain = 0;
+    my $Nback = 0;
+    if (1) {
+	my $ncopies_query = "SELECT ins_id, vol_id, xattr FROM instance JOIN volume USING (vol_id) WHERE (so_id = ${so_id})";
+	
+	my $rowref = $dbh->selectall_arrayref( $ncopies_query );
+	my @array = @$rowref;
+	my $Nrows = @array;
+
+	if ($Nrows == 0) { # We should find this instance
+	    print STDERR "ERROR: NO INSTANCES: $summary\n";
+	    next;
+	}
+
+	## how many xattr = 0 and how many xattr = 1
+	for (my $j = 0; $j < $Nrows; $j ++) {
+	    my $valref = $array[$j];
+	    my @values = @$valref;
+	
+	    my $Nvalue = @values;
+
+	    if ($Nvalue != 3) {
+		print STDERR "ERROR : INVALID DB XATTR RESPONSE: $summary\n";
+		next;
+	    }
+	    my $xattr = $values[2];
+	    
+	    if ($xattr == 0) { $Nmain ++; }
+	    if ($xattr == 1) { $Nback ++; }
+	}
+    }
+
+    # SKIP : did not expect this (already fixed?)
+    if ($Nback > 0) {
+	print STDERR "SKIP: BACKUP EXISTS: $summary\n";
+	next;
+    }
+
+    # single copy @ ITC
+    if (($Nmain == 1) && ($Nback == 0)) {
+	print STDERR "SING: MAKE BACKUP : $summary\n";
+	my $status = 0;
+	$status = vsystem ("neb-replicate $ext_id");
+	if ($status) { print STDERR "FAIL SN1: neb-replicate : $summary\n"; next; }
+	next;
+    }
+
+#XX    # normal duplicate @ ITC
+#XX    if (($Nmain == 2) && ($Nback == 0)) {
+#XX	print STDERR "DUPL: MAKE BACKUP AND CULL : $summary\n";
+#XX	my $status = 0;
+#XX	$status = vsystem("neb-replicate $ext_id");
+#XX	if ($status) { print STDERR "FAIL DU1: neb-replicate : $summary\n"; next; }
+#XX	$status = vsystem("neb-cull $ext_id");
+#XX	if ($status) { print STDERR "FAIL DU2: neb-cull : $summary\n"; next; }
+#XX	next;
+#XX    }
+#XX
+#XX
+#XX    if (($Nmain == 3) && ($Nback == 0)) {
+#XX	print STDERR "TRIP: MAKE BACKUP AND CULL 2X : $summary\n";
+#XX	my $status = 0;
+#XX	$status = vsystem("neb-replicate $ext_id");
+#XX	if ($status) { print STDERR "FAIL TR1: neb-replicate : $summary\n"; next; }
+#XX	$status = vsystem("neb-cull $ext_id");
+#XX	if ($status) { print STDERR "FAIL TR2: neb-cull 1 : $summary\n"; next; }
+#XX	$status = vsystem("neb-cull $ext_id");
+#XX	if ($status) { print STDERR "FAIL TR3: neb-cull 2 : $summary\n"; next; }
+#XX	next;
+#XX    }
+    
+    # multiple copies @ ITC
+    if (($Nmain > 1) && ($Nback == 0)) {
+	my $Ncull = $Nmain - 1;
+	print STDERR "TRIP: MAKE BACKUP AND CULL $Ncull X : $summary\n";
+	my $status = 0;
+	$status = vsystem("neb-replicate $ext_id");
+	if ($status) { print STDERR "FAIL TR1: neb-replicate : $summary\n"; next; }
+	for (my $j = 0; $j < $Ncull; $j ++) {
+	    $status = vsystem("neb-cull $ext_id");
+	    if ($status) { print STDERR "FAIL TRN: neb-cull $j : $summary\n"; next; }
+	}
+	next;
+    }
+    
+    print STDERR "ERROR: NO BACKUP BUT TOO MANY COPIES OR NONE: $summary\n";
+    # all other cases should be handled manually or in the future
+    # if: ($Nmain > 3) && ($Nback == 0) 
+    # call: neb-replicate once & warn
+} # END loop over movefiles
+
+sub vsystem {
+    print STDERR "@_\n";
+    my $status = system ("@_");
+    return $status;
+}
+
Index: trunk/tools/eam/rebalance/rebalance.mana.save
===================================================================
--- trunk/tools/eam/rebalance/rebalance.mana.save	(revision 41509)
+++ trunk/tools/eam/rebalance/rebalance.mana.save	(revision 41509)
@@ -0,0 +1,652 @@
+#!/usr/bin/env mana
+
+# USAGE: rebalance.mana (source) (target) (Ngoal)
+# source : partition name (e.g., /data/ipp070.0/nebulous)
+# target : partition name (e.g., /data/ipp133.0/rebalance/ipp070.0/nebulous)
+# Ngoal  : disk fraction target 
+
+$EXEC = exec
+
+$BIGSIZE = 30M
+$TB_BYTE = 1024^4
+$VERSION = NONE
+
+# $NSUBDIR = 1
+$NSUBDIR = 0x100
+
+# alternatve code for subdirs:
+# list subdirs -glob $source/[0-9,a-f][0-9,a-f]
+# for i 0 $subdirs:n
+
+macro run.find.test
+  if ($0 != 7)
+    echo "USAGE: run.find.test (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) (Fraction)"
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    echo "Fraction : disk fraction target"
+    break
+  end
+
+  local srchost srcvol tgthost tgtvol Fraction
+
+  # NOTE: tgthost and tgtvol are not actually used in this step
+  $srchost = $1
+  $srcvol = $2
+  $tgthost = $3
+  $tgtvol = $4
+  $VERSION = $5
+  $Fraction = $6
+
+  # generate the full paths:
+  $source = /data/$srchost.$srcvol/nebulous
+  # $target = /data/$tgthost.$tgtvol/rebalance/$srchost.$srcvol/nebulous
+
+  ## make sure target exists
+  # mkdir $target
+
+  ## this script must be run on the same machine as the source
+  ## partitions
+
+  $result = `df -l -m $source | tail -n 1`
+  if ("$result" == "") 
+    echo "invalid source partition or partition is not local"
+    echo "source: $source"
+    break
+  end
+
+  list word -split $result
+
+  $Nbytes = 1024*1024*((1 - $Fraction)*$word:1 - $word:3)
+
+  echo "MB total : $word:1";                            # e.g. 100M
+  echo "MB used  : $word:2";                            # e.g.  95M
+  echo "MB avail : $word:3";                            # e.g.   5M
+
+  echo goal used : {$Fraction*$word:1};                 # e.g., 10M
+  echo to remove : {$word:2 - $Fraction*$word:1};       # e.g., 95M - 90M = 5M
+  echo alt calc  : {(1 - $Fraction)*$word:1 - $word:3}; # e.g., 10M - 5M = 5M
+
+  echo Nbytes to remove: $Nbytes
+  echo NGB to remove   : {$Nbytes/1024^3}
+
+  if ($Nbytes < 0)
+    echo "source partition is already below goal fraction"
+    echo "Nbytes to remove : $Nbytes"
+    break
+  end
+
+  # calculate the number of bytes to remove per subdir
+  $NremoveSub = $Nbytes / 256
+
+  echo "** removing {$NremoveSub/$TB_BYTE} TB per nebulous subdir for total of {$Nbytes/$TB_BYTE} TB"
+end
+
+macro run.find
+  if ($0 != 7)
+    echo "USAGE: run.find (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) (Fraction)"
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    echo "Fraction : disk fraction target"
+    break
+  end
+
+  local srchost srcvol tgthost tgtvol Fraction
+
+  # NOTE: tgthost and tgtvol are not actually used in this step
+  $srchost = $1
+  $srcvol = $2
+  $tgthost = $3
+  $tgtvol = $4
+  $VERSION = $5
+  $Fraction = $6
+
+  # generate the full paths:
+  $source = /data/$srchost.$srcvol/nebulous
+  $target = /data/$tgthost.$tgtvol/rebalance/$srchost.$srcvol/nebulous
+
+  ## make sure target exists
+  # mkdir $target
+
+  ## this script must be run on the same machine as the source
+  ## partitions
+
+  $result = `df -l -m $source | tail -n 1`
+  if ("$result" == "") 
+    echo "invalid source partition or partition is not local"
+    echo "source: $source"
+    break
+  end
+
+  list word -split $result
+
+  $Nbytes = 1024*1024*((1 - $Fraction)*$word:1 - $word:3)
+
+  echo "MB total : $word:1";                            # e.g. 100M
+  echo "MB used  : $word:2";                            # e.g.  95M
+  echo "MB avail : $word:3";                            # e.g.   5M
+
+  echo goal used : {$Fraction*$word:1};                 # e.g., 10M
+  echo to remove : {$word:2 - $Fraction*$word:1};       # e.g., 95M - 90M = 5M
+  echo alt calc  : {(1 - $Fraction)*$word:1 - $word:3}; # e.g., 10M - 5M = 5M
+
+  echo Nbytes to remove: $Nbytes
+  echo NGB to remove   : {$Nbytes/1024^3}
+
+  if ($Nbytes < 0)
+    echo "source partition is already below goal fraction"
+    echo "Nbytes to remove : $Nbytes"
+    break
+  end
+
+  # calculate the number of bytes to remove per subdir
+  $NremoveSub = $Nbytes / 256
+
+  echo "** removing {$NremoveSub/$TB_BYTE} TB per nebulous subdir for total of {$Nbytes/$TB_BYTE} TB"
+
+  # list subdirs -glob $source/[0-9,a-f][0-9,a-f]
+
+  for i 0 $NSUBDIR
+    sprintf srcdir "%s/%02x" $source $i
+    sprintf tgtdir "%s/%02x" $target $i
+    # echo $srcdir
+
+    # we save the temp and working files in e.g., /data/ipp070.0/nebulous/00/rebalance
+    mkdir $srcdir/rebalance
+
+    # generate initial file list (all big files)
+    sprintf filelist %s/rebalance/bigfiles.%s.txt $srcdir $VERSION
+    sprintf movelist %s/rebalance/movefiles.%s.txt $srcdir $VERSION
+    sprintf movelog  %s/rebalance/movefiles.%s.log $srcdir $VERSION
+    sprintf subdirset "%s/??" $srcdir
+
+    local isFound
+    file $movelist isFound
+    if ($isFound) continue
+
+    exec find $subdirset -size +$BIGSIZE > $filelist
+
+#   list files -x "/bin/ls -l `cat $filelist` | sort -k 5nr"
+    list files -x "./neb_make_movelist.sh $filelist"
+
+    # generate subset list to match goal size
+    $Ntotal = 0
+    $j = 0
+    while (($Ntotal < $NremoveSub) && ($j < $files:n))
+      list word -split $files:$j
+      $keep:$j = $word:8
+      $Ntotal += $word:4
+      # echo $word:4 $Ntotal : $NremoveSub $keep:$j
+      $j ++
+    end
+    $keep:n = $j
+
+    # did we find enough?
+    if ($Ntotal < $NremoveSub) 
+      echo "WARNING: only found {$Ntotal/$TB_BYTE} TB of {$NremoveSub/$TB_BYTE} TB (increase BIGSIZE)"
+    else
+      echo "found {$Ntotal/$TB_BYTE} TB of {$NremoveSub/$TB_BYTE} TB"
+    end
+
+    # write out the files to a file
+    exec /bin/rm -f $movelist
+    output $movelist
+    for j 0 $keep:n
+      strsub $keep:$j $srcdir/ "" 
+      # echo $keep:$j
+    end
+    output stdout
+
+    echo "saved list in $movelist"
+  end
+end
+
+macro run.rsync
+  if ($0 != 6)
+    echo "USAGE: run.rsync (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    break
+  end
+
+  local srchost srcvol tgthost tgtvol
+
+  $srchost = $1
+  $srcvol = $2
+  $tgthost = $3
+  $tgtvol = $4
+  $VERSION = $5
+
+  # generate the full paths:
+  $source = /data/$srchost.$srcvol/nebulous
+  $target = /data/$tgthost.$tgtvol/rebalance/$srchost.$srcvol/nebulous
+
+  ## make sure target exists
+  mkdir $target
+
+  $TIMEFORMAT = mjd
+  $TIMEREF = 2000/01/01,00:00:00; # this is not used but required to exist for ctimes
+  
+  # give the start of the night in HST hours:
+  $HSTNightStart = 18.5
+  $HSTNightEnd   =  7.5
+
+  $UTNightStart = ($HSTNightStart + 10) / 24; $UTNightStart = $UTNightStart - int($UTNightStart)
+  $UTNightEnd   = ($HSTNightEnd   + 10) / 24; $UTNightEnd   = $UTNightEnd   - int($UTNightEnd)
+
+  for i 0 $NSUBDIR
+    sprintf srcdir "%s/%02x" $source $i
+    sprintf tgtdir "%s/%02x" $target $i
+    # echo $srcdir
+
+    # generate initial file list (all big files)
+    sprintf movelist %s/rebalance/movefiles.%s.txt $srcdir $VERSION
+    sprintf movelog  %s/rebalance/movefiles.%s.log $srcdir $VERSION
+
+    echo "rsyncing $srcdir/ to $tgtdir/ : $movelog"
+    break -auto off
+    exec rsync -auv --files-from=$movelist $srcdir/ $tgtdir/ >& $movelog
+    break -auto on
+
+    # check the time: is it night time (sleep and restart later)
+    $isNight = 1
+    while ($isNight)
+      ctimes -abs now -var tnow
+      $tfrac = $tnow - int($tnow)
+      $isNight = (($tfrac > $UTNightStart) && ($tfrac < $UTNightEnd))
+      if ($isNight) 
+        echo "it is night time, sleeping until the morning"
+        sleep 600
+      end
+    end
+  end
+end
+
+macro run.rsync.check
+  if ($0 != 7)
+    echo "USAGE: run.rsync.check (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) (OUTVER)"
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    break
+  end
+
+  local srchost srcvol tgthost tgtvol
+
+  $srchost = $1
+  $srcvol = $2
+  $tgthost = $3
+  $tgtvol = $4
+  $VERSION = $5
+  $OUTVER  = $6
+
+  # generate the full paths:
+  $source = /data/$srchost.$srcvol/nebulous
+  $target = /data/$tgthost.$tgtvol/rebalance/$srchost.$srcvol/nebulous
+
+  for i 0 $NSUBDIR
+    sprintf srcdir "%s/%02x" $source $i
+    sprintf tgtdir "%s/%02x" $target $i
+    # echo $srcdir
+
+    # generate initial file list (all big files)
+    sprintf movelist %s/rebalance/movefiles.%s.txt $srcdir $VERSION
+    sprintf movelog  %s/rebalance/movefiles.%s.log $srcdir $OUTVER
+
+    echo "rsyncing $srcdir/ to $tgtdir/ : $movelog"
+    break -auto off
+    exec rsync -auv --files-from=$movelist $srcdir/ $tgtdir/ >& $movelog
+    break -auto on
+  end
+end
+
+# names are defined for a single src & tgt pair
+macro run.insert
+  if ($0 != 6)
+    echo "USAGE: run.insert (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    break
+  end
+
+  local srchost tgthost srcvol tgtvol
+
+  $srchost = $1
+  $srcvol = $2
+
+  $tgthost = $3
+  $tgtvol = $4
+  $VERSION = $5
+  
+  $myhost = `hostname`
+  if ("$myhost" != "$tgthost")
+    echo "please run on $tgthost"
+    break
+  end
+
+  # generate the full paths:
+  $source = /data/$srchost.$srcvol/nebulous
+  $target = /data/$tgthost.$tgtvol/rebalance/$srchost.$srcvol/nebulous
+
+  for i 0 $NSUBDIR
+    sprintf dirname "%02x" $i
+
+    sprintf srcdir "%s/%02x" $source $i
+    sprintf tgtdir "%s/%02x" $target $i
+
+    sprintf logname  %s/rebalance/insert.%s.log $srcdir $VERSION
+
+    echo "insert from $tgtdir/ : $logname"
+    $EXEC ./neb_reinsert.pl --tgthost $tgthost --tgtvol $tgtvol --srchost $srchost --srcvol $srcvol --holding rebalance --dir $dirname > $logname
+  end
+
+  $NtotalError = 0
+  for i 0 $NSUBDIR
+    sprintf srcdir "%s/%02x" $source $i
+    sprintf logname  %s/rebalance/insert.%s.log $srcdir $VERSION
+    # count total number of errors
+    $NgroupError = `grep ERROR $logname | wc -l`
+    $NtotalError += $NgroupError
+  end
+
+  if ($NtotalError > 0) 
+    echo "ERROR: $NtotalError errors with insert"
+    echo "use this command to see the errors:"
+    fprint "grep ERROR %s/??/rebalance/insert.%s.log" $source $VERSION
+  else
+    echo "SUCCESS: no insert errors detected"
+  end
+end
+
+# names are defined for a single src & tgt pair
+macro run.insert.check
+  if ($0 != 6)
+    echo "USAGE: run.insert.check (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    break
+  end
+
+  local srchost tgthost srcvol tgtvol
+
+  $srchost = $1
+  $srcvol = $2
+
+  $source = /data/$srchost.$srcvol/nebulous
+
+  $tgthost = $3
+  $tgtvol = $4
+  $VERSION = $5
+  
+  $myhost = `hostname`
+  if ("$myhost" != "$tgthost")
+    echo "please run on $tgthost"
+    break
+  end
+
+  # I have a list of files in 'movelist' which I want to double-check
+  # the instances corresponding to those files should be on tgt not src
+
+  for i 0 $NSUBDIR
+    sprintf srcdir "%s/%02x" $source $i
+    sprintf subdir "%02x" $i
+
+    # generate initial file list (all big files)
+    sprintf movelist %s/rebalance/movefiles.%s.txt $srcdir $VERSION
+    sprintf checklog %s/rebalance/movefiles.%s.chk $srcdir $VERSION
+
+     echo ./neb_check_reinsert.pl --tgthost $tgthost --tgtvol $tgtvol --subdir $subdir --list $movelist > $checklog
+    $EXEC ./neb_check_reinsert.pl --tgthost $tgthost --tgtvol $tgtvol --subdir $subdir --list $movelist > $checklog
+  end
+
+  $NtotalError = 0
+  for i 0 $NSUBDIR
+    sprintf srcdir "%s/%02x" $source $i
+    sprintf logname %s/rebalance/movefiles.%s.chk $srcdir $VERSION
+    # count total number of errors
+    $NgroupError = `grep ERROR $logname | wc -l`
+    $NtotalError += $NgroupError
+  end
+
+  if ($NtotalError > 0) 
+    echo "ERROR: $NtotalError errors with check"
+  else
+    echo "SUCCESS: no check errors detected"
+  end
+end
+
+macro run.trash
+  if ($0 != 6)
+    echo "USAGE: run.trash (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    break
+  end
+
+  local srchost tgthost srcvol tgtvol
+
+  $srchost = $1
+  $srcvol = $2
+
+  $source = /data/$srchost.$srcvol/nebulous
+  $VERSION = $5
+
+  $myhost = `hostname`
+  if ("$myhost" != "$srchost")
+    echo "please run on $srchost"
+    break
+  end
+
+  for i 0 $NSUBDIR
+    sprintf trashdir "/data/%s.%s/nebulous/%02x/trash" $srchost $srcvol $i
+    mkdir $trashdir
+
+    sprintf srcdir "%s/%02x" $source $i
+    # echo $srcdir
+
+    # list of files which have been moved & checked
+    sprintf movelist %s/rebalance/movefiles.%s.txt $srcdir $VERSION
+
+    list files -x "cat $movelist"
+    for j 0 $files:n
+      # move the file to the trash directory
+      echo mv $srcdir/$files:$j $trashdir
+      exec mv $srcdir/$files:$j $trashdir
+    end
+  end
+end
+
+macro run.trash.check
+  if ($0 != 6)
+    echo "USAGE: run.trash.check (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    break
+  end
+
+  local srchost tgthost srcvol tgtvol
+
+  $srchost = $1
+  $srcvol = $2
+
+  $VERSION = $5
+
+  $myhost = `hostname`
+  if ("$myhost" != "$srchost")
+    echo "please run on $srchost"
+    break
+  end
+
+  for i 0 $NSUBDIR
+    sprintf trashdir "/data/%s.%s/nebulous/%02x/trash" $srchost $srcvol $i
+    sprintf savedir  "/data/%s.%s/nebulous/%02x/save" $srchost $srcvol $i
+    sprintf trashlog "/data/%s.%s/nebulous/%02x/rebalance/check.trash.%s.log" $srchost $srcvol $i $VERSION
+
+     echo ./neb_check_trash.pl --srchost $srchost --trashdir $trashdir --savedir $savedir > $trashlog
+    $EXEC ./neb_check_trash.pl --srchost $srchost --trashdir $trashdir --savedir $savedir > $trashlog
+  end
+
+  $NtotalError = 0
+  for i 0 $NSUBDIR
+    sprintf trashlog "/data/%s.%s/nebulous/%02x/rebalance/check.trash.%s.log" $srchost $srcvol $i $VERSION
+    # count total number of errors
+    $NgroupError = `grep ERROR $trashlog | wc -l`
+    $NtotalError += $NgroupError
+  end
+
+  if ($NtotalError > 0) 
+    echo "ERROR: $NtotalError errors with check.trash"
+  else
+    echo "SUCCESS: no check errors detected"
+  end
+end
+
+macro run.empty.trash
+  if ($0 != 6)
+    echo "USAGE: run.empty.trash (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    break
+  end
+
+  local srchost tgthost srcvol tgtvol
+
+  $srchost = $1
+  $srcvol = $2
+
+  $VERSION = $5
+
+  $myhost = `hostname`
+  if ("$myhost" != "$srchost")
+    echo "please run on $srchost"
+    break
+  end
+
+  ### *** I need a test to confirm I have run check.trash *** ####
+
+  for i 0 $NSUBDIR
+    sprintf trashdir "/data/%s.%s/nebulous/%02x/trash" $srchost $srcvol $i
+
+    # list of files which have been moved & checked
+#   sprintf movelist /data/%s.%s/nebulous/%02x/rebalance/movefiles.%s.txt $srchost $srcvol $i $VERSION
+
+    list files -glob $trashdir/*
+    for j 0 $files:n
+      file $files:$j -can-write canWrite
+      if (not($canWrite))
+        echo "SKIP write-protected file : $files:$j"
+        next
+      end    
+      # remove the file from the trash directory
+      echo rm $files:$j
+      exec rm $files:$j
+    end
+  end
+end
+
+if ($SCRIPT) 
+  if ($argv:n < 1) 
+    echo "USAGE: rebalance.mana (MODE) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) [options]"
+    echo ""
+    echo "MODE: find.test, find, rsync, rsync.check, insert, insert.check, trash, trash.check, empty.trash"
+    echo ""
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    echo ""
+    echo "VERSION : free-form string (e.g., YYYYMMDD)"
+    exit 2
+  end
+
+  #### find.test ####
+  if ("$argv:0" == "find.test")
+    if ($argv:n != 7)
+      echo "USAGE: rebalance.mana (find.test) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) (Fraction)"
+      echo "Fraction : disk fraction target"
+      break
+    end
+    run.find.test $argv:1 $argv:2 $argv:3 $argv:4 $argv:5 $argv:6
+    exit 0
+  end
+
+  #### find ####
+  if ("$argv:0" == "find")
+    if ($argv:n != 7)
+      echo "USAGE: rebalance.mana (find) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) (Fraction)"
+      echo "Fraction : disk fraction target"
+      break
+    end
+    run.find $argv:1 $argv:2 $argv:3 $argv:4 $argv:5 $argv:6
+    exit 0
+  end
+
+  #### rsync ####
+  if ("$argv:0" == "rsync")
+    if ($argv:n != 6)
+      echo "USAGE: rebalance.mana (rsync) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+      break
+    end
+    run.rsync $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
+    exit 0
+  end
+
+  #### rsync.check ####
+  if ("$argv:0" == "rsync.check")
+    if ($argv:n != 7)
+      echo "USAGE: rebalance.mana (rsync.check) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) (OUTVER)"
+      break
+    end
+    run.rsync.check $argv:1 $argv:2 $argv:3 $argv:4 $argv:5 $argv:6
+    exit 0
+  end
+
+  #### insert ####
+  if ("$argv:0" == "insert")
+    if ($argv:n != 6)
+      echo "USAGE: rebalance.mana (insert) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+      break
+    end
+    run.insert $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
+    exit 0
+  end
+
+  #### insert.check ####
+  if ("$argv:0" == "insert.check")
+    if ($argv:n != 6)
+      echo "USAGE: rebalance.mana (insert.check) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+      break
+    end
+    run.insert.check $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
+    exit 0
+  end
+
+  #### trash ####
+  if ("$argv:0" == "trash")
+    if ($argv:n != 6)
+      echo "USAGE: rebalance.mana (trash) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+      break
+    end
+    run.trash $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
+    exit 0
+  end
+
+  #### trash.check ####
+  if ("$argv:0" == "trash.check")
+    if ($argv:n != 6)
+      echo "USAGE: rebalance.mana (trash.check) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+      break
+    end
+    run.trash.check $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
+    exit 0
+  end
+
+  #### empty.trash ####
+  if ("$argv:0" == "empty.trash")
+    if ($argv:n != 6)
+      echo "USAGE: rebalance.mana (empty.trash) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+      break
+    end
+    run.empty.trash $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
+    exit 0
+  end
+
+  echo "ERROR: unknown mode $argv:0"
+  exit 1
+end
Index: trunk/tools/eam/rebalance/rsync.check.logs.sh
===================================================================
--- trunk/tools/eam/rebalance/rsync.check.logs.sh	(revision 41509)
+++ trunk/tools/eam/rebalance/rsync.check.logs.sh	(revision 41509)
@@ -0,0 +1,37 @@
+#!/bin/csh
+
+set showlist = 0
+
+if ($#argv >  1) then
+  echo "USAGE: rsync.check.logs.sh [list]"
+  exit 2
+endif
+
+if ($#argv ==  1) then
+  if ("$1" != "list") then
+    echo "USAGE: rsync.check.logs.sh [list]"
+    exit 2
+  endif
+  set showlist = 1
+endif
+
+set myhost = `hostname`
+
+echo "--- volume 0 ---"
+if ($showlist) then
+  cat /data/$myhost.0/nebulous/??/rebalance/movefiles.20201209.chk.log | grep -v "total size" | grep -v "building file list" | grep -v "bytes  received" | grep -v '^../$' | grep -v '^$'
+  cat /data/$myhost.0/nebulous/??/rebalance/movefiles.20201209.chk.log | grep "No such file or directory" 
+else
+  cat /data/$myhost.0/nebulous/??/rebalance/movefiles.20201209.chk.log | grep -v "total size" | grep -v "building file list" | grep -v "bytes  received" | grep -v '^../$' | grep -v '^$' | wc -l
+  cat /data/$myhost.0/nebulous/??/rebalance/movefiles.20201209.chk.log | grep "No such file or directory" | wc -l
+endif
+
+echo "--- volume 1 ---"
+if ($showlist) then
+  cat /data/$myhost.1/nebulous/??/rebalance/movefiles.20201209.chk.log | grep -v "total size" | grep -v "building file list" | grep -v "bytes  received" | grep -v '^../$' | grep -v '^$'
+  cat /data/$myhost.1/nebulous/??/rebalance/movefiles.20201209.chk.log | grep "No such file or directory" 
+else
+  cat /data/$myhost.1/nebulous/??/rebalance/movefiles.20201209.chk.log | grep -v "total size" | grep -v "building file list" | grep -v "bytes  received" | grep -v '^../$' | grep -v '^$' | wc -l
+  cat /data/$myhost.1/nebulous/??/rebalance/movefiles.20201209.chk.log | grep "No such file or directory" | wc -l
+endif
+
Index: trunk/tools/eam/rebalance/run.nebstat.errors
===================================================================
--- trunk/tools/eam/rebalance/run.nebstat.errors	(revision 41509)
+++ trunk/tools/eam/rebalance/run.nebstat.errors	(revision 41509)
@@ -0,0 +1,16 @@
+#!/bin/csh -f
+
+if ($#argv != 1) then
+  echo "USAGE: run.netstat.errors (hostname.vol)"
+  exit 2
+endif
+
+set src = $1
+
+rm -f $src/nebstat.errors.insert.20201209.v0
+
+foreach f (`prcol 8 < $src/errors.insert.20201209.v0`)
+  neb-stat `neb-reverse $f` >> $src/nebstat.errors.insert.20201209.v0
+  echo $f
+end
+