Index: /trunk/tools/eam/shuffle/insert.neb.sh =================================================================== --- /trunk/tools/eam/shuffle/insert.neb.sh (revision 42535) +++ /trunk/tools/eam/shuffle/insert.neb.sh (revision 42535) @@ -0,0 +1,140 @@ +#!/usr/bin/env mana + +$EXEC = exec + +# names are defined for a single src & tgt pair +macro run.ipp130.0 + $srchost = ipp130 + $tgthost = ippb24 + $srcvol = 0 + $tgtvol = 0 + $myhost = `hostname` + if ("$myhost" != "$tgthost") + echo "please run on $tgthost" + break + end + for i 0 0x100 + sprintf dirname "%02x" $i + sprintf logname logs.restore.$srchost/$srchost.$srcvol.%02x $i + mkdir logs.restore.$srchost + $EXEC ./neb_file_restore.pl --tgthost $tgthost --tgtvol $tgtvol --srchost $srchost --srcvol $srcvol --holding backup_rsync --dir $dirname > $logname + end + exit 0 +end + +# names are defined for a single src & tgt pair +macro run.ipp134.0 + $srchost = ipp134 + $tgthost = ippb24 + $srcvol = 0 + $tgtvol = 1 + $myhost = `hostname` + if ("$myhost" != "$tgthost") + echo "please run on $tgthost" + break + end + for i 0 0x100 + sprintf dirname "%02x" $i + sprintf logname logs.restore.$srchost/$srchost.$srcvol.%02x $i + mkdir logs.restore.$srchost + $EXEC ./neb_file_restore.pl --tgthost $tgthost --tgtvol $tgtvol --srchost $srchost --srcvol $srcvol --holding backup_rsync --dir $dirname > $logname + end + exit 0 +end + +# names are defined for a single src & tgt pair +macro run.ipp135.0 + $srchost = ipp135 + $tgthost = ippb25 + $srcvol = 0 + $tgtvol = 0 + $myhost = `hostname` + if ("$myhost" != "$tgthost") + echo "please run on $tgthost" + break + end + for i 0 0x100 + sprintf dirname "%02x" $i + sprintf logname logs.restore.$srchost/$srchost.$srcvol.%02x $i + mkdir logs.restore.$srchost + $EXEC ./neb_file_restore.pl --tgthost $tgthost --tgtvol $tgtvol --srchost $srchost --srcvol $srcvol --holding backup_rsync --dir $dirname > $logname + end + exit 0 +end + +# names are defined for a single src & tgt pair +macro run.ipp140.0 + $srchost = ipp140 + $tgthost = ippb25 + $srcvol = 0 + $tgtvol = 1 + $myhost = `hostname` + if ("$myhost" != "$tgthost") + echo "please run on $tgthost" + break + end + for i 0 0x100 + sprintf dirname "%02x" $i + sprintf logname logs.restore.$srchost/$srchost.$srcvol.%02x $i + mkdir logs.restore.$srchost + $EXEC ./neb_file_restore.pl --tgthost $tgthost --tgtvol $tgtvol --srchost $srchost --srcvol $srcvol --holding backup_rsync --dir $dirname > $logname + end + exit 0 +end + +# names are defined for a single src & tgt pair +macro run.ipp141.0 + $srchost = ipp141 + $tgthost = ippb26 + $srcvol = 0 + $tgtvol = 0 + $myhost = `hostname` + if ("$myhost" != "$tgthost") + echo "please run on $tgthost" + break + end + for i 0 0x100 + sprintf dirname "%02x" $i + sprintf logname logs.restore.$srchost/$srchost.$srcvol.%02x $i + mkdir logs.restore.$srchost + $EXEC ./neb_file_restore.pl --tgthost $tgthost --tgtvol $tgtvol --srchost $srchost --srcvol $srcvol --holding backup_rsync --dir $dirname > $logname + end + exit 0 +end + +# names are defined for a single src & tgt pair +macro run.ipp142.0 + $srchost = ipp142 + $tgthost = ippb26 + $srcvol = 0 + $tgtvol = 1 + $myhost = `hostname` + if ("$myhost" != "$tgthost") + echo "please run on $tgthost" + break + end + for i 0 0x100 + sprintf dirname "%02x" $i + sprintf logname logs.restore.$srchost/$srchost.$srcvol.%02x $i + mkdir logs.restore.$srchost + $EXEC ./neb_file_restore.pl --tgthost $tgthost --tgtvol $tgtvol --srchost $srchost --srcvol $srcvol --holding backup_rsync --dir $dirname > $logname + end + exit 0 +end + +if ($SCRIPT) + if ($argv:n < 1) + echo "USAGE: insert.sh (src)" + break + end + + if ("$argv:0" == "ipp130.0") run.ipp130.0 + if ("$argv:0" == "ipp134.0") run.ipp134.0 + if ("$argv:0" == "ipp135.0") run.ipp135.0 + if ("$argv:0" == "ipp140.0") run.ipp140.0 + if ("$argv:0" == "ipp141.0") run.ipp141.0 + if ("$argv:0" == "ipp142.0") run.ipp142.0 + + echo "unknown volume $argv:0" + exit 1 +end Index: /trunk/tools/eam/shuffle/neb_file_restore.pl =================================================================== --- /trunk/tools/eam/shuffle/neb_file_restore.pl (revision 42535) +++ /trunk/tools/eam/shuffle/neb_file_restore.pl (revision 42535) @@ -0,0 +1,216 @@ +#! /usr/bin/env perl + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# XXX change name "host" to something like tgt? +my ($tgthost,$tgtvol,$srchost,$srcvol,$dir,$holding); + +GetOptions( + 'tgthost=s' => \$tgthost, + 'tgtvol=s' => \$tgtvol, + + 'srchost=s' => \$srchost, + 'srcvol=s' => \$srcvol, + 'dir=s' => \$dir, + 'holding=s' => \$holding) || pod2usage(2 ); + +my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser ); + +my $conffile = './nebulous.config'; + +if (-e $conffile) { +# printf STDERR "found config\n"; + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "$dbname $dbhost $dbport $dbuser $dbpass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless + defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass; + +pod2usage( -msg => "Cannot determine target machine and volume", -exitval => 2) unless + defined $tgthost && defined $tgtvol; + +pod2usage( -msg => "Cannot determine source machine and volume", -exitval => 2) unless + defined $srchost && defined $srcvol; + +pod2usage( -msg => "Cannot determine target directory or holding", -exitval => 2) unless + defined $dir && defined $holding; + +my $host_check = `hostname`; chomp($host_check); + +if ($host_check ne $tgthost) { die "Required to run on $tgthost, not on $host_check\n"; } + +my $time = time(); + +my $dsn; +if ($dbport eq "NONE") { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost"; +} else { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport"; +} + +my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +# example: +# srcpath : /data/ippb01.0/nebulous/00 +# tmppath : /data/ippb22.0/retire_rsync/ippb01.0/nebulous/00 +# tgtpath : /data/ippb22.0/nebulous/00 + +# holding = backup_rsync +# tgthost = ippb20 +# tgtvol = 0 +# srchost = ipp134 +# srcvol = 0 + +## NEED TO CHECK on src and tgt paths +my $tmppath = "/export/${tgthost}.${tgtvol}/$holding/${srchost}.${srcvol}/nebulous.backup/${dir}"; + +my @subdirectories = <$tmppath/*>; + +# Get the vol_id of the (local) destination. + +my $tgt_volume = "${tgthost}.${tgtvol}"; + +# NOTE: we are using special volumes for these backups: +# append "_bck" to the srchost +my $src_volume = "${srchost}_bck.${srcvol}"; + +my $sql_get_volume_id = "SELECT vol_id, name, host, path FROM volume WHERE name = ?"; +my $sql_update_instance = "UPDATE instance SET vol_id = ?, uri = ? WHERE ins_id = ?"; + +my $query = $dbh->prepare( $sql_get_volume_id ); + +$query->execute( $tgt_volume ); +my ($tgt_vol_id, $tgt_vol_name, $tgt_vol_host, $tgt_vol_path) = $query->fetchrow_array; +$query->finish; + +$query->execute( $src_volume ); +my ($src_vol_id, $src_vol_name, $src_vol_host, $src_vol_path) = $query->fetchrow_array; +$query->finish; + +print "LOG INIT $tgthost $tgtvol $srchost $srcvol $dir $tgt_vol_id $src_vol_id\n"; + +foreach my $tmp_subdir (@subdirectories) { + + my $tgt_subdir = $tmp_subdir; + $tgt_subdir =~ s%/$holding/${srchost}.${srcvol}/nebulous.backup%/nebulous%; + +# print "TEST: $tgt_subdir\n"; +# die; + + my @files = <${tmp_subdir}/*>; + my $otime = $time; + + $time = time(); + my $dtime = $time - $otime; + + print "LOG BEGIN $tmp_subdir $time $dtime\n"; + +# these are for testing: +# my $Nfiles = ($#files < 10) ? $#files : 10; +# for (my $i = 0; $i <= $Nfiles; $i++) { + for (my $i = 0; $i <= $#files; $i++) { + + my $tmp_file = $files[$i]; +## print "TMP_FILE: $tmp_file\n"; + + my ($ins_id); + $ins_id = $tmp_file; + $ins_id =~ s/^$tmp_subdir\///; + $ins_id =~ s/\..*//; + + # Write log of what we plan to do. + my $tgt_file = $tmp_file; + $tgt_file =~ s%/$holding/${srchost}.${srcvol}/nebulous.backup%/nebulous%; + + my $db_uri = $tgt_file; + $db_uri =~ s%^/export/%/data/%; + $db_uri = 'file://' . $db_uri; + + my $summary = "$ins_id $tmp_file $tgt_file $db_uri"; + +## print "TGT_FILE: $tgt_file\n"; +## print "INS_ID: $ins_id\n"; +## print "DB_URI: $db_uri\n"; + + # die; + + # Do move of file. + + # check if file exists at new location: + if (-e $tgt_file) { + print "FILE ALREADY FOUND: $summary\n"; + next; + } + + # check that this instance exists in DB (if not, need to add it) + my $validate_ins_id_query = "SELECT ins_id FROM instance WHERE vol_id = ${src_vol_id} AND ins_id = ${ins_id}"; +## print "SQL: $validate_ins_id_query\n"; + if ( $ins_id =~ m/\D/ ) { + print "INVALID ID: $summary\n"; + next; + } + + my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We expect one instance on the source + print "NO DB ENTRY: $summary\n"; + next; + } + + # XXX SKIP actual ops for now + if (0) { + print "mv $tmp_file $tgt_file\n"; + next; + } + + system("mv $tmp_file $tgt_file"); + unless (-e $tgt_file) { die "problem moving file $tmp_file to $tgt_file\n $summary"; } + + # Register database change: + $query = $dbh->prepare( $sql_update_instance ); + $query->execute( $tgt_vol_id, $db_uri, $ins_id ); + $query->finish; + print "GOOD: $summary\n"; + # die; # die after one file + } # END loop over files + rmdir($tmp_subdir); + + # die; # die after one subdir +} # END loop over subdirectories + +rmdir($tmppath); + +# example of using the return value of arrayref: + +# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); +# my @array = @$rowref; +# my $Nrows = @array; + +# print "NROWS: $Nrows\n"; + +# foreach my $valref (@array) { +# my @values = @$valref; +# foreach my $value (@values) { +# print "VAL: $value, "; +# } +# print "\n"; +# } + Index: /trunk/tools/eam/shuffle/ptolemy.rc =================================================================== --- /trunk/tools/eam/shuffle/ptolemy.rc (revision 42535) +++ /trunk/tools/eam/shuffle/ptolemy.rc (revision 42535) @@ -0,0 +1,8 @@ + +PANTASKS_SERVER_STDOUT pantasks.stsci.stdout.log +PANTASKS_SERVER_STDERR pantasks.stsci.stderr.log +PANTASKS_SERVER ippc200 +PASSWORD foobar + +# use a high range to avoid collisions with other pantasks +PANTASKS_SERVER_PORT 2010:2019 Index: /trunk/tools/eam/shuffle/rsync.shuffle.sh =================================================================== --- /trunk/tools/eam/shuffle/rsync.shuffle.sh (revision 42535) +++ /trunk/tools/eam/shuffle/rsync.shuffle.sh (revision 42535) @@ -0,0 +1,155 @@ +# -- sh-mode -- + +# to use this script, load in pantasks and run the init macro first (creates the output directoris) +# Then, issue the command 'run' & the jobs will be quickly generated. issue the command 'setup' to +# create the host targets. + +$backup = 1 +$test = 0 +$delete = + +if (not($?Njob)) set Njob = 0 +if (not($?srcdir:n)) set srcdir:n = 0 +if (not($?tgtdir:n)) set tgtdir:n = 0 +if (not($?srchost:n)) set srchost:n = 0 +if (not($?tgthost:n)) set tgthost:n = 0 + +macro reset + set Njob = 0 + set srcdir:n = 0 + set tgtdir:n = 0 + set srchost:n = 0 + set tgthost:n = 0 +end + +if not($?Njob) set Njob = 0 + +if not($?LOGDIR) + $today = `date +%Y%m%d` + $LOGDIR = logs.$today + mkdir $LOGDIR +end + +macro mkhosts + echo "make hosts manually" + break + for i 0 $srchost:n + control host add $srchost:$i + end +end + +# names are defined for a single src & tgt pair +macro init.ipp130 + for i 0 0x100 + sprint srcname /data/ipp130.0/nebulous.backup/%02x $i + sprint tgtname /data/ippb24.0/backup_rsync/ipp130.0/nebulous.backup/%02x $i + list srcdir -add $srcname + list tgtdir -add $tgtname + + list srchost -add ipp130 + list tgthost -add ippb24 + end +end + +# names are defined for a single src & tgt pair +macro init.ipp134 + for i 0 0x100 + sprint srcname /data/ipp134.0/nebulous.backup/%02x $i + sprint tgtname /data/ippb24.1/backup_rsync/ipp134.0/nebulous.backup/%02x $i + list srcdir -add $srcname + list tgtdir -add $tgtname + + list srchost -add ipp134 + list tgthost -add ippb24 + end +end + +# names are defined for a single src & tgt pair +macro init.ipp135 + for i 0 0x100 + sprint srcname /data/ipp135.0/nebulous.backup/%02x $i + sprint tgtname /data/ippb25.0/backup_rsync/ipp135.0/nebulous.backup/%02x $i + list srcdir -add $srcname + list tgtdir -add $tgtname + + list srchost -add ipp135 + list tgthost -add ippb25 + end +end + +# names are defined for a single src & tgt pair +macro init.ipp140 + for i 0 0x100 + sprint srcname /data/ipp140.0/nebulous.backup/%02x $i + sprint tgtname /data/ippb25.1/backup_rsync/ipp140.0/nebulous.backup/%02x $i + list srcdir -add $srcname + list tgtdir -add $tgtname + + list srchost -add ipp140 + list tgthost -add ippb25 + end +end + +# names are defined for a single src & tgt pair +macro init.ipp141 + for i 0 0x100 + sprint srcname /data/ipp141.0/nebulous.backup/%02x $i + sprint tgtname /data/ippb26.0/backup_rsync/ipp141.0/nebulous.backup/%02x $i + list srcdir -add $srcname + list tgtdir -add $tgtname + + list srchost -add ipp141 + list tgthost -add ippb26 + end +end + +# names are defined for a single src & tgt pair +macro init.ipp142 + for i 0 0x100 + sprint srcname /data/ipp142.0/nebulous.backup/%02x $i + sprint tgtname /data/ippb26.1/backup_rsync/ipp142.0/nebulous.backup/%02x $i + list srcdir -add $srcname + list tgtdir -add $tgtname + + list srchost -add ipp142 + list tgthost -add ippb26 + end +end + +task mkjobs + periods -poll 0.5 + periods -timeout 10 + host none + + task.exec + if (not($?tgtdir:n)) + echo "create targets with init" + periods -exec 2.0 + break + end + + periods -exec 0.2 + if ($Njob >= $tgtdir:n) + periods -exec 20.0 + break + end + + host -required $srchost:$Njob + + stdout $LOGDIR/$srchost:$Njob.$Njob.log + stderr $LOGDIR/$srchost:$Njob.$Njob.log + +# $exclude = --exclude-from=$EXCLUDE + $exclude = + + $run = rsync $exclude --bwlimit=20000 -auv $srcdir:$Njob/ $tgtdir:$Njob/ + + if ($test) + command echo $run + else + command $run + end + + $Njob ++ + end +end