- Timestamp:
- Oct 28, 2016, 4:09:19 PM (10 years ago)
- File:
-
- 1 edited
-
trunk/ippScripts/scripts/rawcheck.pl (modified) (12 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/rawcheck.pl
r37423 r39796 22 22 23 23 24 my ($server,$dbname,$exp_id );24 my ($server,$dbname,$exp_id,$do_cull); 25 25 26 26 $server = $ENV{'NEB_SERVER'} unless $server; … … 30 30 'dbname=s' => \$dbname, 31 31 'exp_id|x=s' => \$exp_id, 32 'cull' => \$do_cull, 32 33 ) || pod2usage( 2 ); 34 35 unless(defined($do_cull)) { 36 $do_cull = 0; 37 } 33 38 34 39 # Option parsing … … 42 47 43 48 # Global options: 49 ## Define the configuration. Ideally, this would be retrieved from the nebulous 50 ## database, but there are some issues with that (such as grouping the b nodes 51 ## into a less restrictive "offsite" location). 44 52 45 53 my $do_ops = 1; 46 my %host_mapping = ( 47 'XY01' => 'ipp010','XY02' => 'ipp011','XY03' => 'ipp006','XY04' => 'ipp054', 48 'XY05' => 'ipp007','XY06' => 'ipp055','XY10' => 'ipp008','XY11' => 'ipp056', 49 'XY12' => 'ipp009','XY13' => 'ipp057','XY14' => 'ipp058','XY15' => 'ipp059', 50 'XY16' => 'ipp012','XY17' => 'ipp060','XY20' => 'ipp013','XY21' => 'ipp014', 51 'XY22' => 'ipp061','XY23' => 'ipp015','XY24' => 'ipp016','XY25' => 'ipp062', 52 'XY26' => 'ipp064','XY27' => 'ipp065','XY30' => 'ipp066','XY31' => 'ipp020', 53 'XY32' => 'ipp063','XY33' => 'ipp021','XY34' => 'ipp023','XY35' => 'ipp013', 54 'XY36' => 'ipp024','XY37' => 'ipp019','XY40' => 'ipp025','XY41' => 'ipp018', 55 'XY42' => 'ipp017','XY43' => 'ipp015','XY44' => 'ipp027','XY45' => 'ipp028', 56 'XY46' => 'ipp029','XY47' => 'ipp030','XY50' => 'ipp031','XY51' => 'ipp032', 57 'XY52' => 'ipp033','XY53' => 'ipp034','XY54' => 'ipp035','XY55' => 'ipp036', 58 'XY56' => 'ipp037','XY57' => 'ipp038','XY60' => 'ipp039','XY61' => 'ipp040', 59 'XY62' => 'ipp041','XY63' => 'ipp042','XY64' => 'ipp043','XY65' => 'ipp044', 60 'XY66' => 'ipp046','XY67' => 'ipp047','XY71' => 'ipp048','XY72' => 'ipp049', 61 'XY73' => 'ipp050','XY74' => 'ipp051','XY75' => 'ipp052','XY76' => 'ipp053'); 62 my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1, 63 'ippb02' => 1, 'ippb03' => 1, 64 'ippb04' => 1, 'ippb05' => 1, 65 'ippb06' => 1 66 ); 67 my %backup_destinations = (# 'ippb04' => 1, # full 68 # 'ippb05' => 1, # full 69 'ippb06' => 1 70 ); 71 my $backup_Nvols = 3; 72 73 # Set up nebulous db interface 54 my $i; 55 56 ## Set up nebulous db interface, and pull processing information to consider. 74 57 my $neb = Nebulous::Client->new( 75 58 proxy => "$server", … … 78 61 unless defined $neb; 79 62 80 # Pull data from the gpc1 database 63 ## This new implementation is somewhat messy, but is more general and adaptable. 64 ## First, we set up a requirement mapping, explaining where we want copies, and 65 ## how many copies we want at each site. 66 my %requirement_map = (); 67 $requirement_map{ITC} = 1; 68 $requirement_map{OFFSITE} = 1; 69 $requirement_map{MRTCB} = 0; 70 71 ## Second, construct a list of volumes, mapped to their site location, using the 72 ## same site locations as in the requirement map. 73 my %volume_map = (); 74 for ($i = 4; $i <= 21; $i++) { 75 my $vol = sprintf("ipp%03d.0",$i); 76 $volume_map{$vol} = 'MRTCB'; 77 } 78 for ($i = 23; $i <= 32; $i++) { 79 my $vol = sprintf("ipp%03d.0",$i); 80 $volume_map{$vol} = 'MRTCB'; 81 } 82 for ($i = 54; $i <= 97; $i++) { 83 my $vol = sprintf("ipp%03d.0",$i); 84 $volume_map{$vol} = 'MRTCB'; 85 } 86 for ($i = 100; $i <= 104; $i++) { 87 my $vol = sprintf("ipp%03d.0",$i); 88 $volume_map{$vol} = 'MRTCB'; 89 $vol = sprintf("ipp%03d.1",$i); 90 $volume_map{$vol} = 'MRTCB'; 91 } 92 for ($i = 105; $i <= 117; $i++) { 93 my $vol = sprintf("ipp%03d.0",$i); 94 $volume_map{$vol} = 'ITC'; 95 $vol = sprintf("ipp%03d.1",$i); 96 $volume_map{$vol} = 'ITC'; 97 } 98 for ($i = 0; $i <= 6; $i++) { 99 if ($i == 6) { next; } # This isn't "offsite", it's with the rest of the maui cluster. 100 my $vol = sprintf("ippb%02d.0",$i); 101 $volume_map{$vol} = 'OFFSITE'; 102 $vol = sprintf("ippb%02d.1",$i); 103 $volume_map{$vol} = 'OFFSITE'; 104 $vol = sprintf("ippb%02d.2",$i); 105 $volume_map{$vol} = 'OFFSITE'; 106 } 107 108 ## Next, get disk space values, and check which hosts are listed as available. 109 my %acceptable_volume = (); 110 my $mounts = $neb->mounts(); 111 foreach my $vol_row (@$mounts) { 112 my ($mount_point, $total, $used, $vol_id, $name, $host, $path, $allocate, $available, $xattr) = @{ $vol_row }; 113 if (($allocate == 1)&&($available == 1)&&( $used / $total < 0.98)) { 114 $acceptable_volume{$name} = 1; 115 } 116 else { 117 print "## $name $allocate $available $used $total\n"; 118 $acceptable_volume{$name} = 0; 119 } 120 121 if (($name eq 'ipp106.0')||($name eq 'ipp106.1')) { 122 $acceptable_volume{$name} = 0; 123 } 124 } 125 126 ## Finally, generate lists containing which volumes are located at which site. 127 ## This allows us to randomly select a volume from the list for the site that 128 ## we plan on replicating to. 129 my %volume_lists = (); 130 foreach my $vol_key (keys %requirement_map) { 131 @{ $volume_lists{$vol_key} } = grep { $acceptable_volume{$_} == 1 } ( 132 grep { $volume_map{$_} eq $vol_key } (keys %volume_map) 133 ); 134 print "$vol_key " . join(' ', @{ $volume_lists{$vol_key} }) . "\n"; 135 136 if ($#{ $volume_lists{$vol_key} } == -1) { 137 die "No acceptable volume found for site $vol_key!\n"; 138 } 139 } 140 141 # die; 142 # Pull data from the gpc1 database about the exposure to consider 81 143 my $verbose = 0; 82 144 my $mdcParser = PS::IPP::Metadata::Config->new; 83 145 146 print("$regtool -processedimfile -exp_id $exp_id -dbname $dbname"); 84 147 my $regtool_cmd = "$regtool -processedimfile -exp_id $exp_id -dbname $dbname"; 85 148 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 92 155 &my_die("Unable to parse metadata from regtool -processedimfile", $exp_id); 93 156 157 my $timer_start = time(); 158 my $timer = time(); 159 print "## rawcheck.pl: $exp_id $dbname $do_cull $do_ops $timer_start\n"; 160 # Loop over the imfiles of this exposure. 94 161 foreach my $imfile (@$imfiles) { 95 162 my $key = $imfile->{uri}; … … 100 167 101 168 if (!(defined($hostname))) { $hostname = 'ipp004'; } 102 103 # if (($data_state ne 'full')||($data_state ne 'compressed')) { next; } ## skip things that aren't full. 104 105 # Do validation 106 107 # neb-stat level handling 169 $timer = time() - $timer_start; 170 print ("\n# $key $data_state $md5sum $hostname $class_id T: $timer\n"); 171 172 ## Get instances, and do validation that they have the correct md5sums. 108 173 my $stat = $neb->stat($key); 109 174 die "nebulous key: $key not found" unless $stat; … … 111 176 die "no instances found" unless $instances; 112 177 113 my $user_copies;114 eval {115 $user_copies = $neb->getxattr($key, "user.copies");116 };117 unless(defined($user_copies)) {118 $user_copies = 1;119 }120 121 my @validation;122 my %md5sum_uniq;123 178 my $existing_copies = 0; 124 125 my @existance; 126 my @md5sums; 179 my $Ngood = 0; 180 my %good_instances = (); 181 my %bad_instances = (); 182 127 183 my @files = map {URI->new($_)->file if $_} @$instances; 128 my @diskvols; 129 my @diskhosts; 130 my @quality; 131 my $Ngood = 0; 132 my $quality_mask = 0; 184 my @validation = (); 133 185 134 186 for (my $i = 0; $i <= $#files; $i++) { 187 my ($instance_exists,$instance_md5sum,$instance_host,$instance_volume,$instance_site, $is_good); 188 ($instance_host,$instance_volume) = parse_volume($files[$i]); 189 $instance_site = $volume_map{$instance_volume}; 190 $is_good = 0; 191 135 192 if (-e $files[$i]) { 136 $ existance[$i]= 1;193 $instance_exists = 1; 137 194 $existing_copies++; 138 $md5sums[$i] = local_md5sum($files[$i]); 139 $md5sum_uniq{$md5sums[$i]} = 1; 140 195 $instance_md5sum = local_md5sum($files[$i]); 196 if ($instance_md5sum eq $md5sum) { 197 push @{ $good_instances{$instance_site} }, $i; 198 $is_good = 1; 199 $Ngood++; 200 } 201 else { 202 push @{ $bad_instances{$instance_site} }, $i; 203 } 141 204 } 142 205 else { 143 $existance[$i] = 0; 144 $md5sums[$i] = 'NON-EXISTANT'; 145 $md5sum_uniq{$md5sums[$i]} = 1; 146 } 147 ($diskhosts[$i],$diskvols[$i]) = parse_volume($files[$i]); 148 $validation[$i] = sprintf("% 3d %32s %s %d %s %d", 149 $existance[$i], 150 $md5sums[$i], 151 $files[$i], 152 $md5sums[$i] eq $md5sum, 153 $diskhosts[$i],$diskvols[$i] 154 ); 155 156 # Pre-parse decisions 157 if ($md5sums[$i] ne $md5sum) { 158 $quality[$i] = 0; 159 } 160 elsif ($existance[$i] == 0) { 161 $quality[$i] = 0; 162 } 163 elsif (is_backup_volume($diskhosts[$i])) { 164 $quality[$i] = 1; 165 $quality_mask = $quality_mask | 1; 166 $Ngood++; 167 } 168 elsif ($diskhosts[$i] eq $host_mapping{$class_id}) { 169 $quality[$i] = 1; 170 $quality_mask = $quality_mask | 2; 171 $Ngood++; 172 } 173 else { 174 $quality[$i] = 2; 175 $Ngood++; 176 } 177 } 178 179 print "\n$key $data_state $md5sum $hostname $class_id\n"; 180 if (1) { 181 182 print 183 "object id: ", @$stat[0], "\n", 184 "key: ", @$stat[1], "\n"; 185 print 186 "epoch: ", @$stat[4], "\n"; 187 print 188 "md5sum count: ", scalar(keys %md5sum_uniq), "\n"; 189 print 190 "requested instances: ", $user_copies, "\n", 191 "available instances: ", @$stat[6], "\n", 192 "existing instances: ", $existing_copies, "\n", 193 "total instances: ", @$stat[7], "\n", 194 "instance location:\n", " " x 4; 195 print 196 join("\n" . " " x 4, @validation), "\n"; 197 } 206 $instance_exists = 0; 207 $instance_md5sum = 'NON-EXISTANT'; 208 push @{ $bad_instances{$instance_site} }, $i; 209 } 210 211 $validation[$i] = sprintf(" % 3d %d %32s %s %s %s\n", 212 $instance_exists,$is_good,$instance_md5sum, 213 $files[$i],$instance_host,$instance_volume); 214 } 215 216 $timer = time() - $timer_start; 217 # object_id ext_id epoch available existing total timer 218 printf("%s %s %s %d %d %d %d\n",@$stat[0],@$stat[1],@$stat[4],@$stat[6],$existing_copies,@$stat[7],$timer); 219 # instance_exists is_good instance_md5sum file instance_host instance_volume 220 my $val_string = join('',@validation); 221 print "$val_string"; 198 222 199 223 200 224 # Decide what to do 225 226 ## This block attempts to find an out-of-nebulous instance with a good md5sum, 227 ## and substitutes it for the 0-th listed instance. This gives us one good 228 ## copy to work with. 201 229 if ($Ngood == 0) { 202 230 # DO something to attempt to fix this. … … 220 248 } 221 249 else { 222 $quality[0] = 1;223 250 print "cp $good_file $files[0]\n"; 224 system("cp $good_file $files[0]"); 251 if ($do_ops) { 252 system("cp $good_file $files[0]"); 253 } 225 254 } 226 255 # Begin my best validation thought … … 234 263 235 264 $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. 236 } 237 238 if ($quality[0] == 0) { # The first instance is bad. 239 # But since we're here, and not up there, there must be at least one good copy. Find it. 240 for (my $i = 0; $i <= $#md5sums; $i++) { 241 if ($md5sums[$i] eq $md5sum) { # Found it. 242 print "cp $files[$i] $files[0]\n"; 243 system("cp $files[$i] $files[0]"); 244 # Begin my best validation thought 245 { 246 my $tmpmd5 = local_md5sum($files[0]); 247 if ($tmpmd5 ne $md5sum) { 248 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 249 } 265 ## We've done work here, so we can't do a cull this iteration. 266 if ($do_cull == 1) { $do_cull = -1; } 267 } 268 269 ## We have more than zero bad copies. We may cull some of these in the future, but we should try to 270 ## leave everything in the best state possible. 271 printf(">> %d %d\n",$Ngood, $#files + 1); 272 if ($Ngood != $#files + 1) { 273 my $good_copy; 274 my $good_copy_index; 275 foreach my $site_key (keys %good_instances) { 276 if ($#{ $good_instances{$site_key} } != -1) { 277 $good_copy_index = $good_instances{$site_key}[0]; 278 $good_copy = $files[$good_copy_index]; 279 last; 280 } 281 } 282 printf(">> GOOD: $good_copy\n"); 283 foreach my $site_key (keys %bad_instances) { 284 foreach my $bad_copy_index (@{ $bad_instances{$site_key} }) { 285 print "cp $good_copy $files[$bad_copy_index]\n"; 286 if ($do_ops) { 287 system("cp $good_copy $files[$bad_copy_index]"); 250 288 } 251 # End my best validation thought. 252 last; # We're done here now. 253 } 254 } 255 } 256 257 if ($Ngood == 1) { # We have only one version 258 if ($quality_mask & 1) { # ANd it's on a backup volume 259 print "neb-replicate $key\n"; 260 if ($do_ops) { 261 $neb->replicate($key) or die "failed to replicate the single valid copy"; 262 if ($@) { die $@; } 263 264 } 265 266 } 267 else { # And it's not, so put one there 268 my $rep_vol = get_random_backup_volume(); 289 my $tmpmd5 = local_md5sum($files[$bad_copy_index]); 290 if ($tmpmd5 ne $md5sum) { 291 ## This isn't super critical, so we don't need to die here. 292 warn "Post-repair md5sum does not match! $tmpmd5 != $md5sum: $files[$bad_copy_index]"; 293 } 294 else { 295 ## success 296 push @{ $good_instances{$site_key} }, $bad_copy_index; 297 } 298 } 299 } 300 ## We've done work here, so we can't do a cull this iteration. 301 if ($do_cull == 1) { $do_cull = -1; } 302 } 303 304 ## We can now attempt to make replicated copies to the sites that require additional copies. 305 foreach my $site_key (keys %requirement_map) { 306 my $have_instances = $#{ $good_instances{$site_key} } + 1; 307 print "## $site_key $have_instances $requirement_map{$site_key}\n"; 308 if ($#{ $good_instances{$site_key} } + 1 < $requirement_map{$site_key}) { 309 my $rep_vol = get_random_site_volume($site_key); 269 310 print "neb-replicate --volume $rep_vol $key\n"; 270 311 if ($do_ops) { … … 277 318 @$uris = map {URI->new($_)->file if $_} @$uris; 278 319 my $tmpmd5 = local_md5sum(${ $uris }[0]); 320 321 my $validation_str = sprintf("% 3d %d %32s %s %s %s", 322 -1,-1,$tmpmd5, 323 ${ $uris }[0],"repl",$rep_vol); 324 print 325 join("\n" . " " x 4, $validation_str), "\n"; 326 279 327 if ($tmpmd5 ne $md5sum) { 280 328 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; … … 282 330 # End my best validation thought. 283 331 } 284 } 285 } 286 else { # N >= 2 287 if (!($quality_mask & 1)) { # no backup copy 288 my $rep_vol = get_random_backup_volume(); 289 print "neb-replicate --volume $rep_vol $key\n"; 290 if ($do_ops) { 291 $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; 292 if ($@) { die $@; } 293 294 # Begin my best validation thought 295 system("sync") == 0 or die "Couldn't sync?"; 296 my $uris = $neb->find_instances($key,$rep_vol); 297 @$uris = map {URI->new($_)->file if $_} @$uris; 298 my $tmpmd5 = local_md5sum(${ $uris }[0]); 299 if ($tmpmd5 ne $md5sum) { 300 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 301 } 302 # End my best validation thought. 303 304 } 305 } 306 if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid. 307 for (my $i = 0; $i <= $#files; $i++) { 308 if ($quality[$i] == 2) { 309 $quality[$i] = 1; 310 last; 311 } 312 } 313 } 314 for (my $i = 0; $i <= $#files; $i++) { 315 # print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n"; 316 if ($quality[$i] != 1) { 317 if ($existance[$i] == 0) { # This disk file doesn't exist. 318 system("touch $files[$i]"); 319 } 320 my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i]; 321 print "neb-cull --volume $cull_vol $key\n"; 322 if ($do_ops) { 323 # The tilde here is to force hard volumes. Don't touch it. 324 $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance"; 325 if ($@) { die "$@"; } 326 } 327 } 328 } 329 } 330 # exit(0); 331 } 332 ## We've done work here, so we can't do a cull this iteration. 333 if ($do_cull == 1) { $do_cull = -1; } 334 } 335 } 336 337 ## Do culls if that's what we were going to do. 338 if ($do_cull == -1) { 339 die "Cull option passed, but files were modified in the scan/repair/replicate phase. Not running cull!\n"; 340 } 341 elsif ($do_cull == 1) { 342 ## At this point, we should have no files in the bad_instances lists, because we've repaired them. 343 foreach my $site_key (keys %good_instances) { 344 if ($#{ $good_instances{$site_key} } + 1 > $requirement_map{$site_key}) { 345 for ($i = $requirement_map{$site_key}; $i <= $#{ $good_instances{$site_key} }; $i++) { 346 my $cull_index = ${ $good_instances{$site_key} }[$i]; 347 my ($instance_host,$instance_volume) = parse_volume($files[$cull_index]); 348 print "neb-cull --volume $instance_volume $key\n"; 349 if ($do_ops) { 350 # The tilde here is to force hard volumes. Don't touch it. 351 # Also: the 2 is a "minimum number of copies" restriction. Let's not be crazy here. 352 $neb->cull($key,"~${instance_volume}",2) or die "failed to cull a superfluous instance"; 353 if ($@) { die "$@"; } 354 } 355 } # End loop over extra instances 356 } # End check for sites with extra instances 357 } # End loop over sites. 358 } # End cull 359 360 } ## End loop over imfiles. 332 361 333 362 sub local_md5sum { … … 350 379 my $filename = shift(@_); 351 380 my $full_volume = (split /\//, $filename)[2]; 352 my ($hostname,$vol_index) = split /\./, $full_volume; # /; 353 return($hostname,$vol_index); 354 } 355 356 sub is_backup_volume { 357 my $hostname = shift(@_); 358 if (exists($backup_hosts{$hostname})) { 359 return(1); 360 } 361 return(0); 362 } 363 364 sub get_random_backup_volume { 365 my $NN = scalar keys %backup_destinations; 366 my $backup_host = (keys %backup_destinations)[int(rand($NN))]; 367 my $backup_vol = int(rand($backup_Nvols)); 368 369 return("${backup_host}.${backup_vol}"); 381 my ($hostname,undef) = split /\./, $full_volume; # /; 382 return($hostname,$full_volume); 383 } 384 385 sub get_random_site_volume { 386 my $site_key = shift(@_); 387 my $NN = scalar @{ $volume_lists{$site_key} }; 388 my $backup_volume = ${ $volume_lists{$site_key} }[int(rand($NN))]; 389 return($backup_volume); 370 390 } 371 391
Note:
See TracChangeset
for help on using the changeset viewer.
