IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 41406 for trunk


Ignore:
Timestamp:
Aug 25, 2020, 10:15:47 AM (6 years ago)
Author:
eugene
Message:

updates to rebalance scripts

Location:
trunk/tools/eam/rebalance
Files:
2 added
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/tools/eam/rebalance/neb_check_trash.pl

    r41384 r41406  
    1212use Getopt::Long qw( GetOptions :config auto_help auto_version );
    1313
    14 my ($srchost,$trashdir);
     14my ($srchost,$trashdir,$savedir);
    1515
    1616GetOptions(
    17     'srchost=s' => \$srchost,
    18     'trashdir=s' => \$trashdir) || pod2usage(2);
     17    'srchost=s'  => \$srchost,
     18    'trashdir=s' => \$trashdir,
     19    'savedir=s'  => \$savedir) || pod2usage(2);
    1920
    2021my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser );
    … …  
    3940    defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass;
    4041
    41 pod2usage( -msg => "Cannot determine trash directory", -exitval => 2) unless defined $trashdir && defined $srchost;
     42pod2usage( -msg => "Cannot determine srchost, trashdir, or savedir", -exitval => 2) unless defined $trashdir && defined $savedir && defined $srchost;
    4243
    4344my  $host_check = `hostname`; chomp($host_check);
    … …  
    5253
    5354my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n";
     55
     56unless (mkdir $savedir) { die "failed to make savedir\n"; }
    5457
    5558my @files = <${trashdir}/*>;
    … …  
    9699
    97100    if ($Nrows == 0) { # We should find this instance
    98         print "ERROR : NO DB ENTRY: $summary\n";
     101        print "ERROR : NO DB ENTRY: (saving) $summary\n";
     102        system ("mv $filename $savedir/");
    99103        next;
    100104    }

  • trunk/tools/eam/rebalance/rebalance.mana

    r41385 r41406  
    158158    sprintf movelist %s/rebalance/movefiles.%s.txt $srcdir $VERSION
    159159    sprintf movelog  %s/rebalance/movefiles.%s.log $srcdir $VERSION
    160     sprintf subdirset "%s/0?" $srcdir
     160    sprintf subdirset "%s/??" $srcdir
    161161    exec find $subdirset -size +$BIGSIZE > $filelist
    162162    list files -x "/bin/ls -l `cat $filelist` | sort -k 5nr"
    … …  
    228228end
    229229
     230macro run.rsync.check
     231  if ($0 != 7)
     232    echo "USAGE: run.rsync.check (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) (OUTVER)"
     233    echo "srchost, tgthost : e.g., ipp070"
     234    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
     235    break
     236  end
     237
     238  local srchost srcvol tgthost tgtvol
     239
     240  $srchost = $1
     241  $srcvol = $2
     242  $tgthost = $3
     243  $tgtvol = $4
     244  $VERSION = $5
     245  $OUTVER  = $6
     246
     247  # generate the full paths:
     248  $source = /data/$srchost.$srcvol/nebulous
     249  $target = /data/$tgthost.$tgtvol/rebalance/$srchost.$srcvol/nebulous
     250
     251  for i 0 $NSUBDIR
     252    sprintf srcdir "%s/%02x" $source $i
     253    sprintf tgtdir "%s/%02x" $target $i
     254    # echo $srcdir
     255
     256    # generate initial file list (all big files)
     257    sprintf movelist %s/rebalance/movefiles.%s.txt $srcdir $VERSION
     258    sprintf movelog  %s/rebalance/movefiles.%s.log $srcdir $OUTVER
     259
     260    echo "rsyncing $srcdir/ to $tgtdir/ : $movelog"
     261    exec rsync -auv --files-from=$movelist $srcdir/ $tgtdir/ >& $movelog
     262  end
     263end
     264
    230265# names are defined for a single src & tgt pair
    231266macro run.insert
    … …  
    287322
    288323# names are defined for a single src & tgt pair
    289 macro run.check
     324macro run.insert.check
    290325  if ($0 != 6)
    291     echo "USAGE: run.check (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
     326    echo "USAGE: run.insert.check (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
    292327    echo "srchost, tgthost : e.g., ipp070"
    293328    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
    … …  
    384419end
    385420
    386 macro run.check.trash
     421macro run.trash.check
    387422  if ($0 != 6)
    388     echo "USAGE: run.check.trash (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
     423    echo "USAGE: run.trash.check (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
    389424    echo "srchost, tgthost : e.g., ipp070"
    390425    echo "srcvol, tgtvol   : e.g., 0 (for /data/ipp070.0/nebulous)"
    … …  
    407442  for i 0 $NSUBDIR
    408443    sprintf trashdir "/data/%s.%s/nebulous/%02x/trash" $srchost $srcvol $i
     444    sprintf savedir  "/data/%s.%s/nebulous/%02x/save" $srchost $srcvol $i
    409445    sprintf trashlog "/data/%s.%s/nebulous/%02x/rebalance/check.trash.%s.log" $srchost $srcvol $i $VERSION
    410446
    411      echo ./neb_check_trash.pl --srchost $srchost --trashdir $trashdir > $trashlog
    412     $EXEC ./neb_check_trash.pl --srchost $srchost --trashdir $trashdir > $trashlog
     447     echo ./neb_check_trash.pl --srchost $srchost --trashdir $trashdir --savedir $savedir > $trashlog
     448    $EXEC ./neb_check_trash.pl --srchost $srchost --trashdir $trashdir --savedir $savedir > $trashlog
    413449  end
    414450
    … …  
    455491
    456492    # list of files which have been moved & checked
    457     sprintf movelist /data/%s.%s/nebulous/%02x/rebalance/movefiles.%s.txt $srchost $srcvol $i $VERSION
    458 
    459     list files -x "cat $movelist"
     493#   sprintf movelist /data/%s.%s/nebulous/%02x/rebalance/movefiles.%s.txt $srchost $srcvol $i $VERSION
     494
     495    list files -glob $trashdir/*
    460496    for j 0 $files:n
    461       basename $files:$j -var myfile
    462 
     497      file $files:$j -can-write canWrite
     498      if (not($canWrite))
     499        echo "SKIP write-protected file : $files:$j"
     500        next
     501      end   
    463502      # remove the file from the trash directory
    464       echo rm $trashdir/$myfile
    465       exec rm $trashdir/$myfile
     503      echo rm $files:$j
     504      exec rm $files:$j
    466505    end
    467506  end
    … …  
    469508
    470509if ($SCRIPT)
    471   if ($argv:n < 6)
     510  if ($argv:n < 1)
    472511    echo "USAGE: rebalance.mana (MODE) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) [options]"
    473512    echo ""
    474     echo "MODE: find.test, find, rsync, insert, check, trash, check.trash, empty.trash"
     513    echo "MODE: find.test, find, rsync, rsync.check, insert, insert.check, trash, trash.check, empty.trash"
    475514    echo ""
    476515    echo "srchost, tgthost : e.g., ipp070"
    … …  
    513552  end
    514553
     554  #### rsync.check ####
     555  if ("$argv:0" == "rsync.check")
     556    if ($argv:n != 7)
     557      echo "USAGE: rebalance.mana (rsync.check) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION) (OUTVER)"
     558      break
     559    end
     560    run.rsync.check $argv:1 $argv:2 $argv:3 $argv:4 $argv:5 $argv:6
     561    exit 0
     562  end
     563
    515564  #### insert ####
    516565  if ("$argv:0" == "insert")
    … …  
    523572  end
    524573
    525   #### check ####
    526   if ("$argv:0" == "check")
     574  #### insert.check ####
     575  if ("$argv:0" == "insert.check")
    527576    if ($argv:n != 6)
    528       echo "USAGE: rebalance.mana (check) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
    529       break
    530     end
    531     run.check $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
     577      echo "USAGE: rebalance.mana (insert.check) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
     578      break
     579    end
     580    run.insert.check $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
    532581    exit 0
    533582  end
    … …  
    543592  end
    544593
    545   #### check.trash ####
    546   if ("$argv:0" == "check.trash")
     594  #### trash.check ####
     595  if ("$argv:0" == "trash.check")
    547596    if ($argv:n != 6)
    548       echo "USAGE: rebalance.mana (check.trash) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
    549       break
    550     end
    551     run.check.trash $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
     597      echo "USAGE: rebalance.mana (trash.check) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
     598      break
     599    end
     600    run.trash.check $argv:1 $argv:2 $argv:3 $argv:4 $argv:5
    552601    exit 0
    553602  end
    … …  
    556605  if ("$argv:0" == "empty.trash")
    557606    if ($argv:n != 6)
    558       echo "USAGE: rebalance.mana (check.trash) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
     607      echo "USAGE: rebalance.mana (empty.trash) (srchost) (srcvol) (tgthost) (tgtvol) (VERSION)"
    559608      break
    560609    end
    … …  
    563612  end
    564613
    565   exit 0
    566 end
     614  echo "ERROR: unknown mode $argv:0"
     615  exit 1
     616end
Note: See TracChangeset for help on using the changeset viewer.