Index: trunk/tools/eam/neb.cleanup.20190201/check.clean.sh =================================================================== --- trunk/tools/eam/neb.cleanup.20190201/check.clean.sh (revision 41573) +++ trunk/tools/eam/neb.cleanup.20190201/check.clean.sh (revision 41573) @@ -0,0 +1,16 @@ +#!/bin/csh -f + +if ($#argv != 2) then + echo "USAGE: check.clean.sh (dir) (mode)" + echo $#argv + exit 2 +endif + +grep MULTI $1/clean.????.txt | wc -l +grep SINGLE $1/clean.????.txt | wc -l +grep -v MULTI $1/clean.????.txt | grep -v SINGLE | wc -l + +if ("$2" == "list") then + grep -v MULTI $1/clean.????.txt | grep -v SINGLE +endif + Index: trunk/tools/eam/neb.cleanup.20190201/check.scan.sh =================================================================== --- trunk/tools/eam/neb.cleanup.20190201/check.scan.sh (revision 41573) +++ trunk/tools/eam/neb.cleanup.20190201/check.scan.sh (revision 41573) @@ -0,0 +1,16 @@ +#!/bin/csh -f + +if ($#argv != 2) then + echo "USAGE: check.scan.sh (dir) (mode)" + echo $#argv + exit 2 +endif + +grep MULTI $1/scan.????.txt | wc -l +grep SINGLE $1/scan.????.txt | wc -l +grep -v MULTI $1/scan.????.txt | grep -v SINGLE | wc -l + +if ("$2" == "list") then + grep -v MULTI $1/scan.????.txt | grep -v SINGLE +endif + Index: trunk/tools/eam/neb.cleanup.20190201/check.status.sh =================================================================== --- trunk/tools/eam/neb.cleanup.20190201/check.status.sh (revision 41573) +++ trunk/tools/eam/neb.cleanup.20190201/check.status.sh (revision 41573) @@ -0,0 +1,29 @@ +#!/bin/csh -f + +set list = "" +set list = "$list ipp008" +set list = "$list ipp020" +set list = "$list ipp023" +set list = "$list ipp024" +set list = "$list ipp025" +set list = "$list ipp026" +set list = "$list ipp027" +set list = "$list ipp028" +set list = "$list ipp029" + +foreach host ($list) + set item = "$host.0" + set Nr = `wc -l $item/range.txt | awk '{print $1 - 1}'` + set Ns = `ls $item/scan.????.txt | wc -l` + set Nc = `ls $item/clean.????.txt | wc -l` + set Ngs = `grep ALLGONE $item/list.scan.txt | wc -l` + set Ngc = `grep ALLGONE $item/list.clean.txt | wc -l` + set Nin = `wc -l $item/clean.????.txt | tail -n 1 | prcol 1` + + set hrs = 0 + if (-f log.$host) then + set hrs = `stat -c "%Y %X" log.$host | awk '{print ($1 - $2)/3600.0}'` + endif + + echo "$item $Nr $Ns $Nc $Ngs $Ngc $hrs $Nin" | awk '{printf "%10s | %4d %4d %4d | %3d %3d | %7.4f %d\n", $1, $2, $3, $4, $5, $6, $7, $8}' +end Index: trunk/tools/eam/neb.cleanup.20190201/neb_file_restore.pl =================================================================== --- trunk/tools/eam/neb.cleanup.20190201/neb_file_restore.pl (revision 41573) +++ trunk/tools/eam/neb.cleanup.20190201/neb_file_restore.pl (revision 41573) @@ -0,0 +1,207 @@ +#! /usr/bin/env perl + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# 2 : simulate for 10 files +# 1 : only try one file (for real) +# 0 : full real run +my $TESTRUN = 0; + +# XXX change name "host" to something like tgt? +my ($tgthost,$tgtvol,$srchost,$srcvol,$dir,$holding); + +GetOptions( + 'tgthost=s' => \$tgthost, + 'tgtvol=s' => \$tgtvol, + + 'srchost=s' => \$srchost, + 'srcvol=s' => \$srcvol, + 'dir=s' => \$dir, + 'holding=s' => \$holding) || pod2usage(2 ); + +my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser ); +my $conffile = '/home/panstarrs/ipp/shuffle.20180614/nebulous.config'; + +if (-e $conffile) { +# printf STDERR "found config\n"; + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "$dbname $dbhost $dbport $dbuser $dbpass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless + defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass; + +pod2usage( -msg => "Cannot determine target machine and volume", -exitval => 2) unless + defined $tgthost && defined $tgtvol; + +pod2usage( -msg => "Cannot determine source machine and volume", -exitval => 2) unless + defined $srchost && defined $srcvol; + +pod2usage( -msg => "Cannot determine target directory or holding", -exitval => 2) unless + defined $dir && defined $holding; + +my $host_check = `hostname`; chomp($host_check); + +if ($host_check ne $tgthost) { die "Required to run on $tgthost, not on $host_check\n"; } + +my $time = time(); + +my $dsn; +if ($dbport eq "NONE") { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost"; +} else { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport"; +} + +my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +# example: +# srcpath : /data/ippb01.0/nebulous/00 +# tmppath : /data/ippb22.0/retire_rsync/ippb01.0/nebulous/00 +# tgtpath : /data/ippb22.0/nebulous/00 + +my $tmppath = "/export/${tgthost}.${tgtvol}/$holding/${srchost}.${srcvol}/nebulous/${dir}"; + +my @subdirectories = <$tmppath/*>; + +# Get the vol_id of the (local) destination. + +my $tgt_volume = "${tgthost}.${tgtvol}"; +my $src_volume = "${srchost}.${srcvol}"; + +my $sql_get_volume_id = "SELECT vol_id, name, host, path FROM volume WHERE name = ?"; +my $sql_update_instance = "UPDATE instance SET vol_id = ?, uri = ? WHERE ins_id = ?"; + +my $query = $dbh->prepare( $sql_get_volume_id ); + +$query->execute( $tgt_volume ); +my ($tgt_vol_id, $tgt_vol_name, $tgt_vol_host, $tgt_vol_path) = $query->fetchrow_array; +$query->finish; + +$query->execute( $src_volume ); +my ($src_vol_id, $src_vol_name, $src_vol_host, $src_vol_path) = $query->fetchrow_array; +$query->finish; + +print "LOG INIT $tgthost $tgtvol $srchost $srcvol $dir $tgt_vol_id $src_vol_id\n"; + +foreach my $tmp_subdir (@subdirectories) { + + my $tgt_subdir = $tmp_subdir; + $tgt_subdir =~ s%/$holding/${srchost}.${srcvol}/nebulous%/nebulous%; + + my @files = <${tmp_subdir}/*>; + my $otime = $time; + + $time = time(); + my $dtime = $time - $otime; + + print "LOG BEGIN $tmp_subdir $time $dtime\n"; + + my $Nfiles = $#files; + if ($TESTRUN) { + $Nfiles = 10; + } + + for (my $i = 0; $i <= $Nfiles; $i++) { + + my $tmp_file = $files[$i]; + my ($ins_id); + $ins_id = $tmp_file; + $ins_id =~ s/^$tmp_subdir\///; + $ins_id =~ s/\..*//; + + # Write log of what we plan to do. + my $tgt_file = $tmp_file; + $tgt_file =~ s%/$holding/${srchost}.${srcvol}/nebulous%/nebulous%; + + my $db_uri = $tgt_file; + $db_uri =~ s%^/export/%/data/%; + $db_uri = 'file://' . $db_uri; + + my $summary = "$ins_id $tmp_file $tgt_file $db_uri"; + + # Do move of file. + + # check if file exists at new location: + if (-e $tgt_file) { + print "FILE ALREADY FOUND: $summary\n"; + next; + } + + # check that this instance exists in DB (if not, need to add it) + my $validate_ins_id_query = "SELECT ins_id FROM instance WHERE vol_id = ${src_vol_id} AND ins_id = ${ins_id}"; + # print "SQL: $validate_ins_id_query\n"; + if ( $ins_id =~ m/\D/ ) { + print "INVALID ID: $summary\n"; + next; + } + + my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We expect one instance on the source + print "NO DB ENTRY: $summary\n"; + next; + } + + # print "PASSED DB CHECK: $#instances\n"; + + # XXX SKIP actual ops for now + if ($TESTRUN == 2) { + print "mv $tmp_file $tgt_file\n"; + print "SQL: $sql_update_instance\n"; + next; + } + + system("mv $tmp_file $tgt_file"); + unless (-e $tgt_file) { die "problem moving file $tmp_file to $tgt_file\n $summary"; } + + # Register database change: + $query = $dbh->prepare( $sql_update_instance ); + $query->execute( $tgt_vol_id, $db_uri, $ins_id ); + $query->finish; + print "GOOD: $summary\n"; + + if ($TESTRUN) { die; } # die after one file + } # END loop over files + + if ($TESTRUN) { die; } + rmdir($tmp_subdir); + + # die; # die after one subdir +} # END loop over subdirectories + +rmdir($tmppath); + +# example of using the return value of arrayref: + +# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); +# my @array = @$rowref; +# my $Nrows = @array; + +# print "NROWS: $Nrows\n"; + +# foreach my $valref (@array) { +# my @values = @$valref; +# foreach my $value (@values) { +# print "VAL: $value, "; +# } +# print "\n"; +# } + Index: trunk/tools/eam/neb.cleanup.20190201/neb_retired_nodes.pl =================================================================== --- trunk/tools/eam/neb.cleanup.20190201/neb_retired_nodes.pl (revision 41573) +++ trunk/tools/eam/neb.cleanup.20190201/neb_retired_nodes.pl (revision 41573) @@ -0,0 +1,436 @@ +#! /usr/bin/env perl + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +my ($mode,$vol_id,$min_id,$max_id); + +GetOptions( + 'mode|m=s' => \$mode, + 'vol_id|v=s' => \$vol_id, + 'min_id|s=s' => \$min_id, + 'max_id|e=s' => \$max_id + ) || pod2usage(2); + +my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser ); +my $conffile = 'nebulous.config'; + +if (-e $conffile) { +# printf STDERR "found config\n"; + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; } + } + close(IN); + # print STDERR "$dbname $dbhost $dbport $dbuser $dbpass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless + defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass; + +pod2usage( -msg => "Cannot determine target volume ID", -exitval => 2) unless defined $vol_id; + +pod2usage( -msg => "Cannot determine mode", -exitval => 2) unless defined $mode; + +my $time = time(); + +my $dsn; +if ($dbport eq "NONE") { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost"; +} else { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport"; +} + +my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +# generate a list of ins_id ranges with at most 50k instances per range +if ($mode eq "subdivide") { + + my $min_id_def = -1; + my $max_id_def = -1; + + if (defined $max_id) { $max_id_def = $max_id; } + if (defined $min_id) { $min_id_def = $min_id; } + + my ($t1, $t2, $t3) = &subdiv_range_recurse ($vol_id, $min_id_def, $max_id_def, 0, 20000); + my @Nins = @$t1; + my @min_id_obs = @$t2; + my @max_id_obs = @$t3; + + printf "%8s | %12s | %12s \n", "N", "min ins_id", "max ins_id"; + for (my $i = 0; $i < @Nins; $i++) { + printf "%8d | %12d | %12d \n", $Nins[$i], $min_id_obs[$i], $max_id_obs[$i]; + } + exit 0; +} + +# single subdivide pass +if ($mode eq "subdiv_basic") { + + my $min_id_def = -1; + my $max_id_def = -1; + + if (defined $max_id) { $max_id_def = $max_id; } + if (defined $min_id) { $min_id_def = $min_id; } + + my ($t1, $t2, $t3) = &subdiv_range ($vol_id, $min_id_def, $max_id_def); + my @Nins = @$t1; + my @min_id_obs = @$t2; + my @max_id_obs = @$t3; + + printf "%8s | %12s | %12s \n", "N", "min ins_id", "max ins_id"; + for (my $i = 0; $i < @Nins; $i++) { + printf "%8d | %12d | %12d \n", $Nins[$i], $min_id_obs[$i], $max_id_obs[$i]; + } + exit 0; +} + +# split the given range into 1/20 range chunks +if ($mode eq "range") { + my ($sql, $result, @row); + + # get the full range of ins_ids for this volume: + $sql = "SELECT count(ins_id), min(ins_id), max(ins_id) FROM instance WHERE vol_id = $vol_id"; + if (defined $max_id) { + $sql .= " and ins_id <= $max_id"; + } + if (defined $min_id) { + $sql .= " and ins_id >= $min_id"; + } + $result = &mysql_select ($sql); + + @row = $result->fetchrow_array(); + my $N_all = $row[0]; + my $min_ins_id = $row[1]; + my $max_ins_id = $row[2]; + + # generate a histogram of the counts in bins of 1/20 of Max_ins_id + my $delta_ins_id = int(($max_ins_id - $min_ins_id) / 20) + 1; + +# printf "%8s | %12s | %12s | %12s | %12s \n", "N", "min so_id", "max so_id", "min ins_id", "max ins_id"; + printf "%8s | %12s | %12s \n", "N", "min ins_id", "max ins_id"; + for (my $i = 0; $i < 20; $i++) { + + # get the full range of ins_ids for this volume: + my $min_ref_id = $i*$delta_ins_id + $min_ins_id; + $sql = "SELECT count(ins_id), min(ins_id), max(ins_id) FROM instance WHERE vol_id = $vol_id and ins_id >= $min_ref_id and ins_id <= ($min_ref_id + $delta_ins_id)"; + $result = &mysql_select ($sql); + @row = $result->fetchrow_array(); + if (defined $row[1]) { +# printf "%8d | %12d | %12d | %12d | %12d \n", $row[0], $row[1], $row[2], $row[3], $row[4]; + printf "%8d | %12d | %12d \n", $row[0], $row[1], $row[2]; + } + } + exit 0; +} + +# assess the state of the existing instances +if ($mode eq "scan") { + pod2usage( -msg => "Cannot determine min or max ins_id", -exitval => 2) unless defined $min_id && defined $max_id; + + my $sql_so_ids = "SELECT so_id FROM instance WHERE vol_id = $vol_id AND ins_id >= $min_id AND ins_id <= $max_id"; + + my $sql_all_so_ids = "SELECT so_id, COUNT(ins_id) as Nall FROM instance JOIN volume USING (vol_id) WHERE so_id IN ($sql_so_ids) GROUP BY so_id"; + my $sql_gone_so_ids = "SELECT so_id, COUNT(ins_id) as Ngone FROM instance JOIN volume USING (vol_id) WHERE so_id IN ($sql_so_ids) AND (xattr = 5) GROUP BY so_id"; + + my $sql_get_so_id_counts = "SELECT ext_id, so_id, Nall, Ngone FROM ($sql_all_so_ids) as t1 JOIN storage_object using (so_id) LEFT JOIN ($sql_gone_so_ids) as t2 using (so_id)"; + + my $result = &mysql_select ($sql_get_so_id_counts); + + while (my @row = $result->fetchrow_array()) { + my $ext_id = $row[0]; + my $so_id = $row[1]; + my $Nall = $row[2]; + my $Ngone = $row[3]; + my $Nkept = $Nall - $Ngone; + + # we have four interesting cases: + if ($Nkept == 0) { + printf "ALLGONE: $ext_id $so_id $Nall $Nkept $Ngone\n"; + next; + } + if ($Ngone == 0) { + printf "ALLKEPT: $ext_id $so_id $Nall $Nkept $Ngone\n"; + next; + } + if ($Nkept == 1) { + printf "SINGLE: $ext_id $so_id $Nall $Nkept $Ngone\n"; + next; + } + if ($Nkept > 1) { + printf "MULTI: $ext_id $so_id $Nall $Nkept $Ngone\n"; + next; + } + printf "UNEXPECTED: $ext_id $so_id $Nall $Nkept $Ngone\n"; + } + exit 0; +} + +# cleanup the instance (delete instances which are SINGLE or MULTI) +if ($mode eq "clean") { + pod2usage( -msg => "Cannot determine min or max ins_id", -exitval => 2) unless defined $min_id && defined $max_id; + + my $sql_so_ids = "SELECT so_id FROM instance WHERE vol_id = $vol_id AND ins_id >= $min_id AND ins_id <= $max_id"; + + my $sql_all_so_ids = "SELECT so_id, COUNT(ins_id) as Nkept FROM instance JOIN volume USING (vol_id) WHERE so_id IN ($sql_so_ids) GROUP BY so_id"; + my $sql_gone_so_ids = "SELECT so_id, COUNT(ins_id) as Nkept FROM instance JOIN volume USING (vol_id) WHERE so_id IN ($sql_so_ids) AND (xattr = 5) GROUP BY so_id"; + + my $sql_get_so_id_counts = "SELECT * FROM ($sql_all_so_ids) as t1 LEFT JOIN ($sql_gone_so_ids) as t2 using (so_id)"; + + my $result = &mysql_select ($sql_get_so_id_counts); + + while (my @row = $result->fetchrow_array()) { + my $so_id = $row[0]; + my $Nall = $row[1]; + my $Ngone = $row[2]; + my $Nkept = $Nall - $Ngone; + + # can we validate that so_id is an integer? + + # we have four interesting cases: + if ($Nkept == 0) { + printf "ALLGONE: $so_id $Nall $Nkept $Ngone\n"; + next; + } + if ($Ngone == 0) { + printf "ALLKEPT: $so_id $Nall $Nkept $Ngone\n"; + next; + } + if ($Nkept == 1) { + + my @ins_id_del = (); + + # list the entries which will be removed + my $sql_gt = "SELECT ext_id, so_id, ins_id, vol_id, xattr from instance join storage_object using (so_id) join volume using (vol_id) where so_id = $so_id and (xattr = 5)"; + my $res_gt = &mysql_select ($sql_gt); + while (my @res_row = $res_gt->fetchrow_array()) { + printf "RMSINGLE: $res_row[0] $res_row[1] $res_row[2] $res_row[3] $res_row[4]\n"; + push @ins_id_del, $res_row[2]; + } + + # remove the entries + foreach my $ins_id (@ins_id_del) { + if (not is_a_valid_number ($ins_id)) { + printf "SKIPPING ins_id $ins_id\n"; + next; + } + my $sql_rm = "DELETE from instance where ins_id = $ins_id"; + my $res_rm = &mysql_select ($sql_rm); + # printf "DEL: $sql_rm\n"; + } + next; + } + if ($Nkept > 1) { + + my @ins_id_del = (); + + # list the entries which will be removed + my $sql_gt = "SELECT ext_id, so_id, ins_id, vol_id, xattr from instance join storage_object using (so_id) join volume using (vol_id) where so_id = $so_id and (xattr = 5)"; + my $res_gt = &mysql_select ($sql_gt); + while (my @res_row = $res_gt->fetchrow_array()) { + printf "RMMULTI: $res_row[0] $res_row[1] $res_row[2] $res_row[3] $res_row[4]\n"; + push @ins_id_del, $res_row[2]; + } + + # remove the entries + foreach my $ins_id (@ins_id_del) { + if (not is_a_valid_number ($ins_id)) { + printf "SKIPPING ins_id $ins_id\n"; + next; + } + my $sql_rm = "DELETE from instance where ins_id = $ins_id"; + my $res_rm = &mysql_select ($sql_rm); + # printf "DEL: $sql_rm\n"; + } + + next; + } + printf "UNEXPECTED: $so_id $Nall $Nkept $Ngone\n"; + } + exit 0; +} + +printf "UNKNOWN mode: $mode\n"; +exit 2; + + +# uses global dbh +sub mysql_select { + + my $query = $_[0]; + + my $statement = $dbh->prepare($query); + + $statement->execute(); + + return $statement; +} + +sub mysql_dump_result { + + my $result = $_[0]; + + my $Nfields = $result->{NUM_OF_FIELDS}; + + print "Nfields: $Nfields\n"; + + for (my $i = 0; $i < $Nfields; $i++) { + printf "Field %d : %s\n", $i, $result->{NAME}->[$i]; + } + + while (my @row = $result->fetchrow_array()) { + for (my $i = 0; $i < $Nfields; $i++) { + print "$row[$i] "; + } + print "\n"; + } + return; +} + +sub subdiv_range { + my $vol_id = $_[0]; + my $min_id = $_[1]; + my $max_id = $_[2]; + + my ($sql, $result, @row); + + # get the full range of ins_ids for this volume: + $sql = "SELECT count(ins_id), min(ins_id), max(ins_id) FROM instance WHERE vol_id = $vol_id"; + + if ($max_id >= 0) { + $sql .= " and ins_id <= $max_id"; + } + + if ($min_id >= 0) { + $sql .= " and ins_id >= $min_id"; + } + $result = &mysql_select ($sql); + + @row = $result->fetchrow_array(); + my $N_all = $row[0]; + my $min_ins_id = $row[1]; + my $max_ins_id = $row[2]; + + # generate a histogram of the counts in bins of 1/20 of Max_ins_id + my $delta_ins_id = int(($max_ins_id - $min_ins_id) / 20) + 1; + + my (@range_Nval, @range_min_id, @range_max_id); + for (my $i = 0; $i < 20; $i++) { + + # get the full range of ins_ids for this volume: + my $min_ref_id = $i*$delta_ins_id + $min_ins_id; + $sql = "SELECT count(ins_id), min(ins_id), max(ins_id) FROM instance WHERE vol_id = $vol_id and ins_id >= $min_ref_id and ins_id <= ($min_ref_id + $delta_ins_id)"; + $result = &mysql_select ($sql); + @row = $result->fetchrow_array(); + if (defined $row[1]) { + printf "%8d | %12d | %12d ** \n", $row[0], $row[1], $row[2]; + push @range_Nval, $row[0]; + push @range_min_id, $row[1]; + push @range_max_id, $row[2]; + } + } + return (\@range_Nval, \@range_min_id, \@range_max_id); +} + +## count the number of instances on this volume in the ins_id range +## in bins 1/20 of the ins_id range. if the number of instances +## in any bin is larger than max_count, recursively split the bin +sub subdiv_range_recurse { + my $vol_id = $_[0]; + my $min_id = $_[1]; + my $max_id = $_[2]; + my $depth = $_[3]; + my $max_count = $_[4]; + + print STDERR "STARTING level $depth\n"; + + my (@range_Nval, @range_min_id, @range_max_id); + + if ($depth > 5) { + print STDERR "ESCAPING after 5 levels\n"; + push @range_Nval, $vol_id; + push @range_min_id, $min_id; + push @range_max_id, $max_id; + return (\@range_Nval, \@range_min_id, \@range_max_id); + } + + my ($sql, $result, @row); + + # get the full range of ins_ids for this volume: + $sql = "SELECT count(ins_id), min(ins_id), max(ins_id) FROM instance WHERE vol_id = $vol_id"; + + if ($max_id >= 0) { + $sql .= " and ins_id <= $max_id"; + } + + if ($min_id >= 0) { + $sql .= " and ins_id >= $min_id"; + } + $result = &mysql_select ($sql); + + @row = $result->fetchrow_array(); + my $N_all = $row[0]; + my $min_ins_id = $row[1]; + my $max_ins_id = $row[2]; + + # generate a histogram of the counts in bins of 1/20 of Max_ins_id + my $delta_ins_id = int(($max_ins_id - $min_ins_id) / 20) + 1; + + for (my $i = 0; $i < 20; $i++) { + + # get the full range of ins_ids for this volume: + my $min_ref_id = $i*$delta_ins_id + $min_ins_id; + $sql = "SELECT count(ins_id), min(ins_id), max(ins_id) FROM instance WHERE vol_id = $vol_id and ins_id >= $min_ref_id and ins_id <= ($min_ref_id + $delta_ins_id)"; + $result = &mysql_select ($sql); + @row = $result->fetchrow_array(); + if (defined $row[1]) { + # printf "%8d | %12d | %12d ** \n", $row[0], $row[1], $row[2]; + if ($row[0] > $max_count) { + my ($t1, $t2, $t3) = &subdiv_range_recurse ($vol_id, $row[1], $row[2], $depth + 1, $max_count); + push @range_Nval, @$t1; + push @range_min_id, @$t2; + push @range_max_id, @$t3; + } else { + push @range_Nval, $row[0]; + push @range_min_id, $row[1]; + push @range_max_id, $row[2]; + } + } + } + return (\@range_Nval, \@range_min_id, \@range_max_id); +} + +sub sub_test { + + my (@array, @ar2); + + push @array, 1; + push @array, 5; + push @array, 3; + + push @ar2, 11; + push @ar2, 52; + push @ar2, 31; + + return (\@array, \@ar2); +} + +sub is_a_valid_number { + + my $x = $_[0]; + + if ($x eq "") { return 0; } + { no warnings; if ($x ne $x+0) { return 0; }} + if ($x < 1) { return 0; } + return 1; +} Index: trunk/tools/eam/neb.cleanup.20190201/run.clean.sh =================================================================== --- trunk/tools/eam/neb.cleanup.20190201/run.clean.sh (revision 41573) +++ trunk/tools/eam/neb.cleanup.20190201/run.clean.sh (revision 41573) @@ -0,0 +1,33 @@ +#!/usr/bin/env mana + +macro go + if ($0 != 4) + echo "USAGE: go (volID) (rangefile) (outdir)" + break + end + + local i volID outdir outfile rangefile + + $volID = $1 + $rangefile = $2 + $outdir = $3 + + data $rangefile + read -skip 1 minID 3 maxID 5 + + for i 0 minID[] + sprintf outfile $outdir/clean.%04d.txt $i + echo ./neb_retired_nodes.pl --mode clean --vol_id $volID --min_id minID[$i] --max_id maxID[$i] > $outfile + exec ./neb_retired_nodes.pl --mode clean --vol_id $volID --min_id minID[$i] --max_id maxID[$i] > $outfile + end +end + +if ($SCRIPT) + if ($argv:n != 3) + echo "USAGE: run.clean.sh (volID) (rangefile) (outdir)" + exit 2 + end + + go $argv:0 $argv:1 $argv:2 + exit 0 +end Index: trunk/tools/eam/neb.cleanup.20190201/run.full =================================================================== --- trunk/tools/eam/neb.cleanup.20190201/run.full (revision 41573) +++ trunk/tools/eam/neb.cleanup.20190201/run.full (revision 41573) @@ -0,0 +1,50 @@ +#!/bin/csh + +if ($#argv != 1) then + echo "USAGE: run.full (volname)" + exit 2 +endif + +set volname = $1 + +set vol_id = `grep $volname retired.vol.txt | grep -v "^#" | awk '{print $2}'` +if ("$vol_id" == "") then + echo "cannot find volname in retired.vol.txt" + exit 2 +endif + +if ($#vol_id > 1) then + echo "volname $volname is not unique in retired.vol.txt:" + grep $volname retired.vol.txt | grep -v "^#" + echo "please try again" + exit 3 +endif + +set today = `date +%Y%m%d` + + +mkdir -p $volname + +echo ./neb_retired_nodes.pl --mode subdivide --vol_id $vol_id + ./neb_retired_nodes.pl --mode subdivide --vol_id $vol_id > $volname/range.txt + +echo ./run.scan.sh $vol_id $volname/range.txt $volname + ./run.scan.sh $vol_id $volname/range.txt $volname >& $volname/log.scan.$today + +echo ./check.scan.sh $volname list + ./check.scan.sh $volname list >& $volname/list.scan.txt + +set Ngone = `grep ALLGONE $volname/list.scan.txt | wc -l` +if ($Ngone > 100) then + echo "WARNING: too many missing files, giving up" + exit 1 +endif + +echo ./run.clean.sh $vol_id $volname/range.txt $volname + ./run.clean.sh $vol_id $volname/range.txt $volname >& $volname/log.clean.$today + +echo ./check.clean.sh $volname list + ./check.clean.sh $volname list >& $volname/list.clean.txt + +chmod -w $volname +exit 0 Index: trunk/tools/eam/neb.cleanup.20190201/run.group =================================================================== --- trunk/tools/eam/neb.cleanup.20190201/run.group (revision 41573) +++ trunk/tools/eam/neb.cleanup.20190201/run.group (revision 41573) @@ -0,0 +1,23 @@ +#!/bin/csh + +# ./run.full ipp013.0 >& log.ipp013 +# ./run.full ipp014.0 >& log.ipp014 +# ./run.full ipp015.0 >& log.ipp015 +# ./run.full ipp016.0 >& log.ipp016 +# ./run.full ipp017.0 >& log.ipp017 +# ./run.full ipp018.0 >& log.ipp018 +# ./run.full ipp019.0 >& log.ipp019 +# ./run.full ipp020.0 >& log.ipp020 +# ./run.full ipp021.0 >& log.ipp021 + +./run.full ipp008.0 >& log.ipp008 +./run.full ipp020.0 >& log.ipp020 +./run.full ipp023.0 >& log.ipp023 +./run.full ipp024.0 >& log.ipp024 +./run.full ipp025.0 >& log.ipp025 +./run.full ipp026.0 >& log.ipp026 +./run.full ipp027.0 >& log.ipp027 +./run.full ipp028.0 >& log.ipp028 +./run.full ipp029.0 >& log.ipp029 +./run.full ipp030.0 >& log.ipp030 +./run.full ipp031.0 >& log.ipp031 Index: trunk/tools/eam/neb.cleanup.20190201/run.scan.sh =================================================================== --- trunk/tools/eam/neb.cleanup.20190201/run.scan.sh (revision 41573) +++ trunk/tools/eam/neb.cleanup.20190201/run.scan.sh (revision 41573) @@ -0,0 +1,33 @@ +#!/usr/bin/env mana + +macro go + if ($0 != 4) + echo "USAGE: go (volID) (rangefile) (outdir)" + break + end + + local i volID outdir outfile rangefile + + $volID = $1 + $rangefile = $2 + $outdir = $3 + + data $rangefile + read -skip 1 minID 3 maxID 5 + + for i 0 minID[] + sprintf outfile $outdir/scan.%04d.txt $i + echo ./neb_retired_nodes.pl --mode scan --vol_id $volID --min_id minID[$i] --max_id maxID[$i] > $outfile + exec ./neb_retired_nodes.pl --mode scan --vol_id $volID --min_id minID[$i] --max_id maxID[$i] > $outfile + end +end + +if ($SCRIPT) + if ($argv:n != 3) + echo "USAGE: run.scan.sh (volID) (rangefile) (outdir)" + exit 2 + end + + go $argv:0 $argv:1 $argv:2 + exit 0 +end