Index: trunk/ippScripts/scripts/permcheck.pl =================================================================== --- trunk/ippScripts/scripts/permcheck.pl (revision 39899) +++ trunk/ippScripts/scripts/permcheck.pl (revision 39900) @@ -25,5 +25,5 @@ my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); -my ($server,$dbname,$stage,$stage_id); +my ($server,$dbname,$stage,$stage_id,$do_cull,$save_log); $server = $ENV{'NEB_SERVER'} unless $server; @@ -34,5 +34,11 @@ 'stage=s' => \$stage, 'stage_id|x=s' => \$stage_id, + 'save_log' => \$save_log, + 'cull' => \$do_cull, ) || pod2usage( 2 ); + +unless(defined($do_cull)) { + $do_cull = 0; +} # Option parsing @@ -47,37 +53,18 @@ unless defined $stage_id; +if ($save_log) { + my $time = time(); + my $logDest = "neb://any/perm_check/${stage}/${stage}_${stage_id}.${time}"; + my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n"; + $ipprc->redirect_output($logDest) or die "Could not redirect output to logfile ${logDest}\n"; +} + + # Global options: - +## Define the configuration. Ideally, this would be retrieved from the nebulous +## database, but there are some issues with that (such as grouping the b nodes +## into a less restrictive "offsite" location). my $do_ops = 1; -my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1, - 'ippb02' => 1, 'ippb03' => 1, - 'ippb04' => 1, 'ippb05' => 1, - 'ippb06' => 1 - ); -my %backup_destinations = (#'ippb04' => 1, - #'ippb05' => 1, - 'ippb06' => 1 -); - -if ($stage eq 'warp') { # These things are destined to have their only copy on the stsciXX nodes. - %backup_hosts = ('stsci00' => 1, 'stsci01' => 1, 'stsci02' => 1, - #'stsci03' => 1, - 'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1, - 'stsci07' => 1, 'stsci08' => 1, 'stsci09' => 1, - 'stsci10' => 1, 'stsci11' => 1, 'stsci12' => 1, - 'stsci13' => 1, 'stsci14' => 1, 'stsci15' => 1, 'stsci16' => 1, - 'stsci17' => 1, 'stsci18' => 1, 'stsci19' => 1); - %backup_destinations = %backup_hosts; -} - -my $backup_Nvols = 3; - -my $ipprc; -if ($dbname eq 'gpc1') { - $ipprc = PS::IPP::Config->new( "GPC1" ); -} -else { - die "Unknown camera to use."; -} +my $i; # Set up nebulous db interface @@ -88,9 +75,93 @@ unless defined $neb; +## This new implementation is somewhat messy, but is more general and adaptable. +## First, we set up a requirement mapping, explaining where we want copies, and +## how many copies we want at each site. +my %requirement_map = (); +$requirement_map{ITC} = 1; +$requirement_map{OFFSITE} = 1; +$requirement_map{MRTCB} = 0; + +## Second, construct a list of volumes, mapped to their site location, using the +## same site locations as in the requirement map. +my %volume_map = (); +for ($i = 4; $i <= 21; $i++) { + my $vol = sprintf("ipp%03d.0",$i); + $volume_map{$vol} = 'MRTCB'; +} +for ($i = 23; $i <= 32; $i++) { + my $vol = sprintf("ipp%03d.0",$i); + $volume_map{$vol} = 'MRTCB'; +} +for ($i = 54; $i <= 97; $i++) { + my $vol = sprintf("ipp%03d.0",$i); + $volume_map{$vol} = 'MRTCB'; +} +for ($i = 100; $i <= 104; $i++) { + my $vol = sprintf("ipp%03d.0",$i); + $volume_map{$vol} = 'MRTCB'; + $vol = sprintf("ipp%03d.1",$i); + $volume_map{$vol} = 'MRTCB'; +} +for ($i = 105; $i <= 117; $i++) { +# if ($i == 115) { next; } + my $vol = sprintf("ipp%03d.0",$i); + $volume_map{$vol} = 'ITC'; + $vol = sprintf("ipp%03d.1",$i); + $volume_map{$vol} = 'ITC'; +} +for ($i = 0; $i <= 15; $i++) { + if ($i == 6) { next; } # This isn't "offsite", it's with the rest of the maui cluster. + if ($i == 9) { next; } # Not online + + my $vol = sprintf("ippb%02d.0",$i); + $volume_map{$vol} = 'OFFSITE'; + $vol = sprintf("ippb%02d.1",$i); + $volume_map{$vol} = 'OFFSITE'; + $vol = sprintf("ippb%02d.2",$i); + $volume_map{$vol} = 'OFFSITE'; +} + +## Next, get disk space values, and check which hosts are listed as available. +my %acceptable_volume = (); +my $mounts = $neb->mounts(); +foreach my $vol_row (@$mounts) { + my ($mount_point, $total, $used, $vol_id, $name, $host, $path, $allocate, $available, $xattr) = @{ $vol_row }; + if (($allocate == 1)&&($available == 1)&&( $used / $total < 0.98)) { + $acceptable_volume{$name} = 1; + } + else { + print "## $name $allocate $available $used $total\n"; + $acceptable_volume{$name} = 0; + } + if ($name =~ /ippb05/) { + $acceptable_volume{$name} = 0; + } +# if (($name eq 'ipp106.0')||($name eq 'ipp106.1')) { +# $acceptable_volume{$name} = 0; +# } +} + +## Finally, generate lists containing which volumes are located at which site. +## This allows us to randomly select a volume from the list for the site that +## we plan on replicating to. +my %volume_lists = (); +foreach my $vol_key (keys %requirement_map) { + @{ $volume_lists{$vol_key} } = grep { $acceptable_volume{$_} == 1 } ( + grep { $volume_map{$_} eq $vol_key } (keys %volume_map) + ); + print "$vol_key " . join(' ', @{ $volume_lists{$vol_key} }) . "\n"; + + if ($#{ $volume_lists{$vol_key} } == -1) { + die "No acceptable volume found for site $vol_key!\n"; + } +} + # Pull data from the gpc1 database my $verbose = 0; my $mdcParser = PS::IPP::Metadata::Config->new; -my $files; +# Not technically imfiles, but the nebulous check block already uses files as a variable. +my $imfiles; if ($stage eq 'camera') { my $cmd = "$camtool -processedexp -cam_id $stage_id -dbname $dbname"; @@ -101,5 +172,5 @@ &my_die("Unable to perform stagetool: $error_code", $stage_id); } - $files = $mdcParser->parse_list(join "", @$stdout_buf) or + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata from stagetool", $stage_id); } @@ -112,5 +183,5 @@ &my_die("Unable to perform stagetool: $error_code", $stage_id); } - $files = $mdcParser->parse_list(join "", @$stdout_buf) or + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata from stagetool", $stage_id); } @@ -123,5 +194,5 @@ &my_die("Unable to perform stagetool: $error_code", $stage_id); } - $files = $mdcParser->parse_list(join "", @$stdout_buf) or + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata from stagetool", $stage_id); } @@ -134,279 +205,308 @@ &my_die("Unable to perform stagetool: $error_code", $stage_id); } - $files = $mdcParser->parse_list(join "", @$stdout_buf) or + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata from stagetool", $stage_id); } - -my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'], +elsif ($stage eq 'ff') { + my $cmd = "$fftool -result --ff_id $stage_id -dbname $dbname"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stagetool: $error_code", $stage_id); + } + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stagetool", $stage_id); +} + + +my $timer_start = time(); +my $timer = time(); +my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK','PSPHOT.BACKMDL','PSPHOT.PSF.RAW.SAVE'], 'warp' => ['PSWARP.OUTPUT','PSWARP.OUTPUT.MASK','PSWARP.OUTPUT.VARIANCE'], 'stack' => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP', 'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'], - 'skycal' => ['PSASTRO.OUTPUT.CMF']); - - -foreach my $entry (@$files) { + 'skycal' => ['PSASTRO.OUTPUT.CMF','PSPHOT.OUTPUT.CFF'], + 'ff' => ['PSPHOT.OUT.CMF.MEF','PSPHOT.OUTPUT.CFF','PSPHOT.FULLFORCE.OUTPUT']); + +my $timer_start = time(); +my $timer = time(); + +print "## permcheck.pl: $stage $stage_id $dbname $do_cull $do_ops $timer_start\n"; +foreach my $entry (@$imfiles) { my $path_base = $entry->{path_base}; my $data_state = $entry->{state}; my $hostname = $entry->{hostname}; my $quality = $entry->{quality}; + + $timer = $time() - $timer_start; + print "# $path_base $data_state $hostname $quality T: $timer\n"; if ($quality != 0) { next; } - print "# $path_base $data_state $hostname $quality\n"; + + my @keys = (); foreach my $product (@{ $components{$stage} }) { - my @keys = (); - if (($stage eq 'camera')&&($product eq 'PSASTRO.OUTPUT.MASK')) { - my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06', - 'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17', - 'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27', - 'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37', - 'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47', - 'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57', - 'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67', - 'XY71','XY72','XY73','XY74','XY75','XY76'); - foreach my $ota (@otas) { - push @keys, $ipprc->filename($product,$path_base,$ota); - } - } + if ($stage eq 'camera') { + if ($product eq 'PSASTRO.OUTPUT.MASK') { + my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06', + 'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17', + 'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27', + 'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37', + 'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47', + 'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57', + 'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67', + 'XY71','XY72','XY73','XY74','XY75','XY76'); + foreach my $ota (@otas) { + push @keys, $ipprc->filename($product,$path_base,$ota); + } + } + elsif (($product eq 'PSPHOT.BACKMDL')||($product eq 'PSPHOT.PSF.RAW.SAVE')) { + my $chip_id = $entry->{chip_id}; + my $chip_cmd = "$chiptool -processedimfile -chip_id $chip_id -dbname $dbname"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $chip_cmd, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stagetool: $error_code", $stage_id); + } + my $chip_$files = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stagetool", $stage_id); + + foreach my $chip_entry (@$chip_files) { + my $chip_path_base = $chip_entry->{path_base}; + my $class_id = $chip_entry->{class_id}; + push @keys, $ipprc->filename($product,$chip_path_base,$class_id); + } + } + } + elsif (($stage eq 'ff')&&($product eq 'PSPHOT.FULLFORCE.OUTPUT')) { + my $ffsum_cmd = "$fftool -summary -ff_id $stage_id -dbname $dbname"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $ffsum_cmd, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stagetool: $error_code", $stage_id); + } + my $chip_$files = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stagetool", $stage_id); + + foreach my $chip_entry (@$chip_files) { + my $chip_path_base = $chip_entry->{path_base}; + my $class_id = $chip_entry->{class_id}; + push @keys, $ipprc->filename($product,$chip_path_base,); + } + } else { push @keys, $ipprc->filename($product,$path_base); } # Do validation + } + + $timer = time() - $timer_start; + + printf("# Identified %d : %s\n",$#keys + 1, $timer); + foreach my $key (@keys) { + # neb-stat level handling + my $stat = $neb->stat($key); + die "nebulous key: $key not found" unless $stat; + my $instances; + my $md5sum = ''; + + # This needs to be in an eval, because although we expect things to exist, they + # may not. This is a fatal error in rawcheck, but need not be here. + eval { + $instances = $neb->find_instances($key, 'any'); + }; + unless (defined($instances)) { print "## skipping due to zero instances\n"; next; } + die "no instances found" unless $instances; - foreach my $key (@keys) { - # neb-stat level handling - my $stat = $neb->stat($key); - die "nebulous key: $key not found" unless $stat; - my $instances; - eval { - $instances = $neb->find_instances($key, 'any'); - }; - unless (defined($instances)) { print "## skipping due to zero instances\n"; next; } - die "no instances found" unless $instances; + my $Ngood; + my %good_instances = (); + my %bad_instances = (); + + my @files = map {URI->new($_)->file if $_} @$instances; + my @validation = (); + + for (my $i = 0; $i <= $#files; $i++) { + my ($instance_exists,$instance_md5sum,$instance_host,$instance_volume,$instance_site, $is_good); + ($instance_host,$instance_volume) = parse_volume($files[$i]); + $instance_site = $volume_map{$instance_volume}; + $is_good = 0; + + if (-e $files[$i]) { + $instance_exists = 1; + $existing_copies++; + $instance_md5sum = local_md5sum($files[$i]); + + # This is bad, but I don't know what the right solution is. We don't have + # the md5sum a priori. I think this is also the only major change needed + # from rawcheck.pl + if (($md5sum eq '')&&($instance_md5sum ne 'd41d8cd98f00b204e9800998ecf8427e')) { + $md5sum = $instance_md5sum; + } + + if ($instance_md5sum eq $md5sum) { + push @{ $good_instance{$instance_site} }, $i; + $is_good = 1; + $Ngood++; + } + else { + push @{ $bad_instances{$instance_site} }, $i; + } + } + else { + $instance_exists = 0; + $instance_md5sum = 'NON-EXISTANT'; + push @{ $bad_instances{$instance_site} }, $i; + } - my $user_copies; - eval { - $user_copies = $neb->getxattr($key, "user.copies"); - }; - unless(defined($user_copies)) { - $user_copies = 1; - } - - my $md5sum; - my @validation; - my %md5sum_uniq; - my $existing_copies = 0; - - my @existance; - my @md5sums; - my @diskfiles = map {URI->new($_)->file if $_} @$instances; - my @diskvols; - my @diskhosts; - my @quality; - my $Ngood = 0; - my $quality_mask = 0; - - for (my $i = 0; $i <= $#diskfiles; $i++) { - if (-e $diskfiles[$i]) { - $existance[$i] = 1; - $existing_copies++; - $md5sums[$i] = local_md5sum($diskfiles[$i]); - $md5sum_uniq{$md5sums[$i]} = 1; - } - else { - $existance[$i] = 0; - $md5sums[$i] = 'NON-EXISTANT'; - $md5sum_uniq{$md5sums[$i]} = 1; - } - ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]); - $validation[$i] = sprintf("% 3d %32s %s %s %d", - $existance[$i], - $md5sums[$i], - $diskfiles[$i], - $diskhosts[$i],$diskvols[$i] - ); - - # Pre-parse decisions - if ($existance[$i] == 0) { - $quality[$i] = 0; - } - elsif (is_backup_volume($diskhosts[$i])) { - $quality[$i] = 1; - $quality_mask = $quality_mask | 1; - $Ngood++; - } - else { - $quality[$i] = 2; - $Ngood++; - } - } - if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth. - die "There are multiple md5sum values for $key"; - } - $md5sum = (keys(%md5sum_uniq))[0]; - - print "\n$key $data_state $md5sum $hostname\n"; - if (1) { - - print - "object id: ", @$stat[0], "\n", - "key: ", @$stat[1], "\n"; - print - "epoch: ", @$stat[4], "\n"; - print - "md5sum count: ", scalar(keys %md5sum_uniq), "\n"; - print - "requested instances: ", $user_copies, "\n", - "available instances: ", @$stat[6], "\n", - "existing instances: ", $existing_copies, "\n", - "total instances: ", @$stat[7], "\n", - "instance location:\n", " " x 4; - print - join("\n" . " " x 4, @validation), "\n"; - } - + $validation[$i] = sprintf(" % 3d %d %32s %s %s %s\n", + $instance_exists,$is_good,$instance_md5sum, + $files[$i],$instance_host,$instance_volume); + } + + $time = time() - $timer_start; + # object_id ext_id epoch available existing total timer + printf("%s %s %s %d %d %d %d\n",@$stat[0],@$stat[1],@$stat[4],@$stat[6],$existing_copies,@$stat[7],$timer); + # instance_exists is_good instance_md5sum file instance_host instance_volume + my $val_string = join('',@validation); + print "$val_string"; + # Decide what to do - if ($Ngood == 0) { - # DO something to attempt to fix this. - my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; - - open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); - my $good_file = ''; - while (
) { - $_ =~ s/^\s+//; - my ($z,undef,$ff) = split /\s+/; - if (($ff)&&(-e $ff)) { - my $md_response = `md5sum $ff`; - if ($md_response =~ /$md5sum/) { - $good_file = (split /\s+/,$md_response)[1]; - } + + ## This block attempts to find an out-of-nebulous instance with a good md5sum, + ## and substitutes it for the 0-th listed instance. This gives us one good + ## copy to work with. + if ($Ngood == 0) { + # DO something to attempt to fix this. + my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; + + open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); + my $good_file = ''; + while (
) { + $_ =~ s/^\s+//; + my ($z,undef,$ff) = split /\s+/; + if (($ff)&&(-e $ff)) { + my $md_response = `md5sum $ff`; + if ($md_response =~ /$md5sum/) { + $good_file = (split /\s+/,$md_response)[1]; } } - close(DD); - if ($good_file eq '') { - die "No valid instance of key: $key"; - } - else { - $quality[0] = 1; - print "cp $good_file $diskfiles[0]\n"; - vsystem("cp $good_file $diskfiles[0]"); - } - # Begin my best validation thought - { - my $tmpmd5 = local_md5sum($diskfiles[0]); + } + close(DD); + if ($good_file eq '') { + die "No valid instance of key: $key"; + } + else { + print "cp $good_file $files[0]\n"; + if ($do_ops) { + system("cp $good_file $files[0]"); + } + } + # Begin my best validation thought + { + my $tmpmd5 = local_md5sum($files[0]); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + } + # End my best validation thought. + + $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. + ## We've done work here, so we can't do a cull this iteration. + if ($do_cull == 1) { $do_cull = -1; } + } + ## We have more than zero bad copies. We may cull some of these in the future, but we should try to + ## leave everything in the best state possible. + printf(">> %d %d\n",$Ngood, $#files + 1); + if ($Ngood != $#files + 1) { + my $good_copy; + my $good_copy_index; + foreach my $site_key (keys %good_instances) { + if ($#{ $good_instances{$site_key} } != -1) { + $good_copy_index = $good_instances{$site_key}[0]; + $good_copy = $files[$good_copy_index]; + last; + } + } + printf(">> GOOD: $good_copy\n"); + foreach my $site_key (keys %bad_instances) { + foreach my $bad_copy_index (@{ $bad_instances{$site_key} }) { + print "cp $good_copy $files[$bad_copy_index]\n"; + if ($do_ops) { + system("cp $good_copy $files[$bad_copy_index]"); + } + my $tmpmd5 = local_md5sum($files[$bad_copy_index]); + if ($tmpmd5 ne $md5sum) { + ## This isn't super critical, so we don't need to die here. + warn "Post-repair md5sum does not match! $tmpmd5 != $md5sum: $files[$bad_copy_index]"; + } + else { + ## success + push @{ $good_instances{$site_key} }, $bad_copy_index; + } + } + } + ## We've done work here, so we can't do a cull this iteration. + if ($do_cull == 1) { $do_cull = -1; } + } + + ## We can now attempt to make replicated copies to the sites that require additional copies. + foreach my $site_key (keys %requirement_map) { + my $have_instances = $#{ $good_instances{$site_key} } + 1; + print "## $site_key $have_instances $requirement_map{$site_key}\n"; + if ($#{ $good_instances{$site_key} } + 1 < $requirement_map{$site_key}) { + my $rep_vol = get_random_site_volume($site_key); + print "neb-replicate --volume $rep_vol $key\n"; + if ($do_ops) { + $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; + if ($@) { die $@; } + + # Begin my best validation thought + system("sync") == 0 or die "Couldn't sync?"; + my $uris = $neb->find_instances($key,$rep_vol); + @$uris = map {URI->new($_)->file if $_} @$uris; + my $tmpmd5 = local_md5sum(${ $uris }[0]); + + my $validation_str = sprintf("% 3d %d %32s %s %s %s", + -1,-1,$tmpmd5, + ${ $uris }[0],"repl",$rep_vol); + print + join("\n" . " " x 4, $validation_str), "\n"; + if ($tmpmd5 ne $md5sum) { die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; } - } - # End my best validation thought. - - $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. - } - - if ($quality[0] == 0) { # The first instance is bad. - # But since we're here, and not up there, there must be at least one good copy. Find it. - for (my $i = 0; $i <= $#md5sums; $i++) { - if ($md5sums[$i] eq $md5sum) { # Found it. - print "cp $diskfiles[$i] $diskfiles[0]\n"; - vsystem("cp $diskfiles[$i] $diskfiles[0]"); - # Begin my best validation thought - { - my $tmpmd5 = local_md5sum($diskfiles[0]); - if ($tmpmd5 ne $md5sum) { - die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; - } - } - # End my best validation thought. - last; # We're done here now. - } - } - } - - if ($Ngood == 1) { # We have only one version - if ($quality_mask & 1) { # ANd it's on a backup volume - if ($user_copies > 1) { # And we want more than one copy. - print "neb-replicate $key\n"; - if ($do_ops) { - $neb->replicate($key) or die "failed to replicate the single valid copy"; - if ($@) { die $@; } - } - } - } - else { # And it's not, so put one there - my $rep_vol = get_random_backup_volume(); - print "neb-replicate --volume $rep_vol $key\n"; - if ($do_ops) { - $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; - if ($@) { die $@; } - - # Begin my best validation thought - vsystem("sync") == 0 or die "Couldn't sync?"; - my $uris = $neb->find_instances($key,$rep_vol); - @$uris = map {URI->new($_)->file if $_} @$uris; - my $tmpmd5 = local_md5sum(${ $uris }[0]); - if ($tmpmd5 ne $md5sum) { - die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; - } - # End my best validation thought. - } - } - } - else { # N >= 2 - if (!($quality_mask & 1)) { # no backup copy - my $rep_vol = get_random_backup_volume(); - print "neb-replicate --volume $rep_vol $key\n"; - if ($do_ops) { - $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; - if ($@) { die $@; } - - # Begin my best validation thought - vsystem("sync") == 0 or die "Couldn't sync?"; - my $uris = $neb->find_instances($key,$rep_vol); - @$uris = map {URI->new($_)->file if $_} @$uris; - my $tmpmd5 = local_md5sum(${ $uris }[0]); - if ($tmpmd5 ne $md5sum) { - die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; - } - # End my best validation thought. - } - } - if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid. - for (my $i = 0; $i <= $#diskfiles; $i++) { - if ($quality[$i] == 2) { - $quality[$i] = 1; - last; - } - } - } - } -#XXX NEW DEBUG THIS: This should iterate over diskfiles that are not marked with a quality = 2, and cull them. -# For the N>=2, we set that for the primary, and -# for both cases, we do not have a diskfile entry for the newly replicated copy. -# Therefore, this should cull down to the correct number. - for (my $i = 0; $i <= $#diskfiles; $i++) { -# print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n"; - if ($quality[$i] != 1) { - if ($existance[$i] == 0) { # This disk file doesn't exist. - vsystem("touch $diskfiles[$i]"); - } - my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i]; - print "neb-cull --volume $cull_vol $key\n"; - if ($do_ops) { - # The tilde here is to force hard volumes. Don't touch it. - $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance"; - if ($@) { die "$@"; } - } - } - } - # } - } # end keys for this product - } # end product for this entry -} # end entry for this id - -sub vsystem { - my $cmd = shift; - print "$cmd\n"; - if ($do_ops) { - system($cmd); - } -} + # End my best validation thought. + } + ## We've done work here, so we can't do a cull this iteration. + if ($do_cull == 1) { $do_cull = -1; } + } + } + + ## Do culls if that's what we were going to do. + if ($do_cull == -1) { + die "Cull option passed, but files were modified in the scan/repair/replicate phase. Not running cull!\n"; + } + elsif ($do_cull == 1) { + ## At this point, we should have no files in the bad_instances lists, because we've repaired them. + foreach my $site_key (keys %good_instances) { + if ($#{ $good_instances{$site_key} } + 1 > $requirement_map{$site_key}) { + for ($i = $requirement_map{$site_key}; $i <= $#{ $good_instances{$site_key} }; $i++) { + my $cull_index = ${ $good_instances{$site_key} }[$i]; + my ($instance_host,$instance_volume) = parse_volume($files[$cull_index]); + print "neb-cull --volume $instance_volume $key\n"; + if ($do_ops) { + # The tilde here is to force hard volumes. Don't touch it. + # Also: the 2 is a "minimum number of copies" restriction. Let's not be crazy here. + $neb->cull($key,"~${instance_volume}",2) or die "failed to cull a superfluous instance"; + if ($@) { die "$@"; } + } + } # End loop over extra instances + } # End check for sites with extra instances + } # End loop over sites. + } # End cull + +} ## End loop over imfiles. + sub local_md5sum { @@ -425,27 +525,16 @@ } - - sub parse_volume { my $filename = shift(@_); my $full_volume = (split /\//, $filename)[2]; - my ($hostname,$vol_index) = split /\./, $full_volume; # /; - return($hostname,$vol_index); -} - -sub is_backup_volume { - my $hostname = shift(@_); - if (exists($backup_hosts{$hostname})) { - return(1); - } - return(0); -} - -sub get_random_backup_volume { - my $NN = scalar keys %backup_destinations; - my $backup_host = (keys %backup_destinations)[int(rand($NN))]; - my $backup_vol = int(rand($backup_Nvols)); - - return("${backup_host}.${backup_vol}"); + my ($hostname,undef) = split /\./, $full_volume; # /; + return($hostname,$full_volume); +} + +sub get_random_site_volume { + my $site_key = shift(@_); + my $NN = scalar @{ $volume_lists{$site_key} }; + my $backup_volume = ${ $volume_lists{$site_key} }[int(rand($NN))]; + return($backup_volume); }