Changeset 41508 for trunk

Timestamp:

Feb 25, 2021, 4:33:46 PM (5 years ago)

Author:

eugene

Message:

updates to rebalance code

Location:

trunk/tools/eam/rebalance

Files:

• . (copied) (copied from trunk/tools/eam/rebalance )
• neb_check_reinsert.pl (modified) (1 diff)
• neb_check_trash.pl (modified) (1 diff)
• rebalance.mana (modified) (6 diffs)

trunk/tools/eam/rebalance/neb_check_reinsert.pl

@@ -157 +157 @@
     my $vol_id_actual = $values[0];
 
-    ## print "NROWS: $Nrows\n";
-    ## foreach my $valref (@array) {
-    ##  my @values = @$valref;
-    ##  foreach my $value (@values) {
-    ##      print "VAL: $value, ";
-    ##  }
-    ##  print "\n";
-    ## }
-
     if ($vol_id_actual != $tgt_vol_id) {
         print "ERROR : INVALID VOL_ID: $summary\n";

trunk/tools/eam/rebalance/neb_check_trash.pl

@@ -54 +54 @@
 my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n";
 
-unless (mkdir $savedir) { die "failed to make savedir\n"; }
+if (-e $savedir) {
+    unless (-d $savedir) { die "savedir $savedir exists but is not a directory\n"; }
+} else {
+    unless (mkdir $savedir) { die "failed to make savedir\n"; }
+}
 
 my @files = <${trashdir}/*>;

trunk/tools/eam/rebalance/rebalance.mana

@@ -261 +261 @@
 
     echo "rsyncing $srcdir/ to $tgtdir/ : $movelog"
+    break -auto off
     exec rsync -auv --files-from=$movelist $srcdir/ $tgtdir/ >& $movelog
+    break -auto on
   end
 end
@@ -388 +390 @@
   end
 
-  local srchost tgthost srcvol tgtvol
+  local srchost tgthost srcvol tgtvol isFound myfile
 
   $srchost = $1
@@ -415 +417 @@
     for j 0 $files:n
       # move the file to the trash directory
-      echo mv $srcdir/$files:$j $trashdir
-      exec mv $srcdir/$files:$j $trashdir
+      
+      $myfile = $srcdir/$files:$j
+      file $myfile isFound
+      if (not($isFound)) 
+        echo "$myfile not found, skipping"
+        continue
+      end
+      echo mv $myfile $trashdir
+      exec mv $myfile $trashdir
     end
   end
@@ -463 +472 @@
   else
     echo "SUCCESS: no check errors detected"
+  end
+end
+
+# names are defined for a single src & tgt pair
+macro run.location.check
+  if ($0 != 6)
+    echo "USAGE: run.location.check (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    break
+  end
+
+  local srchost tgthost srcvol tgtvol
+
+  $srchost = $1
+  $srcvol = $2
+
+  $source = /data/$srchost.$srcvol/nebulous
+
+  $tgthost = $3
+  $tgtvol = $4
+  $VERSION = $5
+  
+  $myhost = `hostname`
+  if ("$myhost" != "$tgthost")
+    echo "please run on $tgthost"
+    break
+  end
+
+  # I have a list of files in 'movelist' for which I want to check copy locations
+  # if the neb-entries have number of user copies != 2, the file is OK
+  # if the neb-entries is >= 2, one should be on a backup node
+
+  for i 0 $NSUBDIR
+    sprintf srcdir "%s/%02x" $source $i
+    sprintf subdir "%02x" $i
+
+    # generate initial file list (all big files)
+    sprintf movelist %s/rebalance/movefiles.%s.txt $srcdir $VERSION
+    sprintf checklog %s/rebalance/movefiles.%s.loc $srcdir $VERSION
+
+     echo ./neb_check_location.pl --tgthost $tgthost --tgtvol $tgtvol --subdir $subdir --list $movelist > $checklog
+    $EXEC ./neb_check_location.pl --tgthost $tgthost --tgtvol $tgtvol --subdir $subdir --list $movelist > $checklog
+  end
+end
+
+# names are defined for a single src & tgt pair
+macro run.location.fix
+  if ($0 != 6)
+    echo "USAGE: run.location.fix (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+    echo "srchost, tgthost : e.g., ipp070"
+    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
+    break
+  end
+
+  local srchost tgthost srcvol tgtvol
+
+  $srchost = $1
+  $srcvol = $2
+
+  $source = /data/$srchost.$srcvol/nebulous
+
+  $tgthost = $3
+  $tgtvol = $4
+  $VERSION = $5
+  
+  $myhost = `hostname`
+  if ("$myhost" != "$tgthost")
+    echo "please run on $tgthost"
+    break
+  end
+
+  # I have a list of files in 'movelist' for which I want to check copy locations
+  # if the neb-entries have number of user copies != 2, the file is OK
+  # if the neb-entries is >= 2, one should be on a backup node
+
+  for i 0 $NSUBDIR
+    sprintf srcdir "%s/%02x" $source $i
+    sprintf subdir "%02x" $i
+
+    # generate initial file list (all big files)
+    sprintf movelist %s/rebalance/movefiles.%s.txt $srcdir $VERSION
+    sprintf checklog %s/rebalance/movefiles.%s.loc $srcdir $VERSION
+
+     echo ./neb_check_location.pl --tgthost $tgthost --tgtvol $tgtvol --subdir $subdir --list $movelist > $checklog
   end
 end
@@ -513 +607 @@
     echo "USAGE: rebalance.mana (MODE) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) [options]"
     echo ""
-    echo "MODE: find.test, find, rsync, rsync.check, insert, insert.check, trash, trash.check, empty.trash"
+    echo "MODE: find.test, find, rsync, rsync.check, insert, insert.check, location.check, trash, trash.check, empty.trash"
     echo ""
     echo "srchost, tgthost : e.g., ipp070"
@@ -604 +698 @@
   end
 
+  #### location.check ####
+  if ("$argv:0" == "location.check")
+    if ($argv:n != 6)
+      echo "USAGE: rebalance.mana (location.check) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+      break
+    end
+    run.location.check $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
+    exit 0
+  end
+
+  #### location.fix ####
+  if ("$argv:0" == "location.fix")
+    if ($argv:n != 6)
+      echo "USAGE: rebalance.mana (location.fix) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
+      break
+    end
+    run.location.fix $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
+    exit 0
+  end
+
   #### empty.trash ####
   if ("$argv:0" == "empty.trash")