Changeset 39813
- Timestamp:
- Nov 16, 2016, 5:20:24 PM (10 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
tags/ipp-20150312/ippScripts/scripts/rawcheck.pl
- Property svn:mergeinfo set to
r37423 r39813 22 22 23 23 24 my ($server,$dbname,$exp_id );24 my ($server,$dbname,$exp_id,$do_cull,$save_log); 25 25 26 26 $server = $ENV{'NEB_SERVER'} unless $server; … … 30 30 'dbname=s' => \$dbname, 31 31 'exp_id|x=s' => \$exp_id, 32 'save_log' => \$save_log, 33 'cull' => \$do_cull, 32 34 ) || pod2usage( 2 ); 35 36 unless(defined($do_cull)) { 37 $do_cull = 0; 38 } 33 39 34 40 # Option parsing … … 41 47 unless defined $exp_id; 42 48 49 if ($save_log) { 50 my $time = time(); 51 my $logDest = "neb://any/raw_check/exp_${exp_id}.${time}"; 52 my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n"; 53 $ipprc->redirect_output($logDest) or die "Could not redirect output to logfile ${logDest}\n"; 54 } 55 43 56 # Global options: 57 ## Define the configuration. Ideally, this would be retrieved from the nebulous 58 ## database, but there are some issues with that (such as grouping the b nodes 59 ## into a less restrictive "offsite" location). 44 60 45 61 my $do_ops = 1; 46 my %host_mapping = ( 47 'XY01' => 'ipp010','XY02' => 'ipp011','XY03' => 'ipp006','XY04' => 'ipp054', 48 'XY05' => 'ipp007','XY06' => 'ipp055','XY10' => 'ipp008','XY11' => 'ipp056', 49 'XY12' => 'ipp009','XY13' => 'ipp057','XY14' => 'ipp058','XY15' => 'ipp059', 50 'XY16' => 'ipp012','XY17' => 'ipp060','XY20' => 'ipp013','XY21' => 'ipp014', 51 'XY22' => 'ipp061','XY23' => 'ipp015','XY24' => 'ipp016','XY25' => 'ipp062', 52 'XY26' => 'ipp064','XY27' => 'ipp065','XY30' => 'ipp066','XY31' => 'ipp020', 53 'XY32' => 'ipp063','XY33' => 'ipp021','XY34' => 'ipp023','XY35' => 'ipp013', 54 'XY36' => 'ipp024','XY37' => 'ipp019','XY40' => 'ipp025','XY41' => 'ipp018', 55 'XY42' => 'ipp017','XY43' => 'ipp015','XY44' => 'ipp027','XY45' => 'ipp028', 56 'XY46' => 'ipp029','XY47' => 'ipp030','XY50' => 'ipp031','XY51' => 'ipp032', 57 'XY52' => 'ipp033','XY53' => 'ipp034','XY54' => 'ipp035','XY55' => 'ipp036', 58 'XY56' => 'ipp037','XY57' => 'ipp038','XY60' => 'ipp039','XY61' => 'ipp040', 59 'XY62' => 'ipp041','XY63' => 'ipp042','XY64' => 'ipp043','XY65' => 'ipp044', 60 'XY66' => 'ipp046','XY67' => 'ipp047','XY71' => 'ipp048','XY72' => 'ipp049', 61 'XY73' => 'ipp050','XY74' => 'ipp051','XY75' => 'ipp052','XY76' => 'ipp053'); 62 my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1, 63 'ippb02' => 1, 'ippb03' => 1, 64 'ippb04' => 1, 'ippb05' => 1, 65 'ippb06' => 1 66 ); 67 my %backup_destinations = (# 'ippb04' => 1, # full 68 # 'ippb05' => 1, # full 69 'ippb06' => 1 70 ); 71 my $backup_Nvols = 3; 72 73 # Set up nebulous db interface 62 my $i; 63 64 ## Set up nebulous db interface, and pull processing information to consider. 74 65 my $neb = Nebulous::Client->new( 75 66 proxy => "$server", … … 78 69 unless defined $neb; 79 70 80 # Pull data from the gpc1 database 71 ## This new implementation is somewhat messy, but is more general and adaptable. 72 ## First, we set up a requirement mapping, explaining where we want copies, and 73 ## how many copies we want at each site. 74 my %requirement_map = (); 75 $requirement_map{ITC} = 1; 76 $requirement_map{OFFSITE} = 1; 77 $requirement_map{MRTCB} = 0; 78 79 ## Second, construct a list of volumes, mapped to their site location, using the 80 ## same site locations as in the requirement map. 81 my %volume_map = (); 82 for ($i = 4; $i <= 21; $i++) { 83 my $vol = sprintf("ipp%03d.0",$i); 84 $volume_map{$vol} = 'MRTCB'; 85 } 86 for ($i = 23; $i <= 32; $i++) { 87 my $vol = sprintf("ipp%03d.0",$i); 88 $volume_map{$vol} = 'MRTCB'; 89 } 90 for ($i = 54; $i <= 97; $i++) { 91 my $vol = sprintf("ipp%03d.0",$i); 92 $volume_map{$vol} = 'MRTCB'; 93 } 94 for ($i = 100; $i <= 104; $i++) { 95 my $vol = sprintf("ipp%03d.0",$i); 96 $volume_map{$vol} = 'MRTCB'; 97 $vol = sprintf("ipp%03d.1",$i); 98 $volume_map{$vol} = 'MRTCB'; 99 } 100 for ($i = 105; $i <= 117; $i++) { 101 # if ($i == 115) { next; } 102 my $vol = sprintf("ipp%03d.0",$i); 103 $volume_map{$vol} = 'ITC'; 104 $vol = sprintf("ipp%03d.1",$i); 105 $volume_map{$vol} = 'ITC'; 106 } 107 for ($i = 0; $i <= 6; $i++) { 108 if ($i == 6) { next; } # This isn't "offsite", it's with the rest of the maui cluster. 109 my $vol = sprintf("ippb%02d.0",$i); 110 $volume_map{$vol} = 'OFFSITE'; 111 $vol = sprintf("ippb%02d.1",$i); 112 $volume_map{$vol} = 'OFFSITE'; 113 $vol = sprintf("ippb%02d.2",$i); 114 $volume_map{$vol} = 'OFFSITE'; 115 } 116 117 ## Next, get disk space values, and check which hosts are listed as available. 118 my %acceptable_volume = (); 119 my $mounts = $neb->mounts(); 120 foreach my $vol_row (@$mounts) { 121 my ($mount_point, $total, $used, $vol_id, $name, $host, $path, $allocate, $available, $xattr) = @{ $vol_row }; 122 if (($allocate == 1)&&($available == 1)&&( $used / $total < 0.98)) { 123 $acceptable_volume{$name} = 1; 124 } 125 else { 126 print "## $name $allocate $available $used $total\n"; 127 $acceptable_volume{$name} = 0; 128 } 129 if ($name =~ /ippb05/) { 130 $acceptable_volume{$name} = 0; 131 } 132 # if (($name eq 'ipp106.0')||($name eq 'ipp106.1')) { 133 # $acceptable_volume{$name} = 0; 134 # } 135 } 136 137 ## Finally, generate lists containing which volumes are located at which site. 138 ## This allows us to randomly select a volume from the list for the site that 139 ## we plan on replicating to. 140 my %volume_lists = (); 141 foreach my $vol_key (keys %requirement_map) { 142 @{ $volume_lists{$vol_key} } = grep { $acceptable_volume{$_} == 1 } ( 143 grep { $volume_map{$_} eq $vol_key } (keys %volume_map) 144 ); 145 print "$vol_key " . join(' ', @{ $volume_lists{$vol_key} }) . "\n"; 146 147 if ($#{ $volume_lists{$vol_key} } == -1) { 148 die "No acceptable volume found for site $vol_key!\n"; 149 } 150 } 151 152 # die; 153 # Pull data from the gpc1 database about the exposure to consider 81 154 my $verbose = 0; 82 155 my $mdcParser = PS::IPP::Metadata::Config->new; 83 156 157 print("$regtool -processedimfile -exp_id $exp_id -dbname $dbname"); 84 158 my $regtool_cmd = "$regtool -processedimfile -exp_id $exp_id -dbname $dbname"; 85 159 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 92 166 &my_die("Unable to parse metadata from regtool -processedimfile", $exp_id); 93 167 168 my $timer_start = time(); 169 my $timer = time(); 170 print "## rawcheck.pl: $exp_id $dbname $do_cull $do_ops $timer_start\n"; 171 # Loop over the imfiles of this exposure. 94 172 foreach my $imfile (@$imfiles) { 95 173 my $key = $imfile->{uri}; … … 100 178 101 179 if (!(defined($hostname))) { $hostname = 'ipp004'; } 102 103 # if (($data_state ne 'full')||($data_state ne 'compressed')) { next; } ## skip things that aren't full. 104 105 # Do validation 106 107 # neb-stat level handling 180 $timer = time() - $timer_start; 181 print ("\n# $key $data_state $md5sum $hostname $class_id T: $timer\n"); 182 183 ## Get instances, and do validation that they have the correct md5sums. 108 184 my $stat = $neb->stat($key); 109 185 die "nebulous key: $key not found" unless $stat; … … 111 187 die "no instances found" unless $instances; 112 188 113 my $user_copies;114 eval {115 $user_copies = $neb->getxattr($key, "user.copies");116 };117 unless(defined($user_copies)) {118 $user_copies = 1;119 }120 121 my @validation;122 my %md5sum_uniq;123 189 my $existing_copies = 0; 124 125 my @existance; 126 my @md5sums; 190 my $Ngood = 0; 191 my %good_instances = (); 192 my %bad_instances = (); 193 127 194 my @files = map {URI->new($_)->file if $_} @$instances; 128 my @diskvols; 129 my @diskhosts; 130 my @quality; 131 my $Ngood = 0; 132 my $quality_mask = 0; 195 my @validation = (); 133 196 134 197 for (my $i = 0; $i <= $#files; $i++) { 198 my ($instance_exists,$instance_md5sum,$instance_host,$instance_volume,$instance_site, $is_good); 199 ($instance_host,$instance_volume) = parse_volume($files[$i]); 200 $instance_site = $volume_map{$instance_volume}; 201 $is_good = 0; 202 135 203 if (-e $files[$i]) { 136 $ existance[$i]= 1;204 $instance_exists = 1; 137 205 $existing_copies++; 138 $md5sums[$i] = local_md5sum($files[$i]); 139 $md5sum_uniq{$md5sums[$i]} = 1; 140 206 $instance_md5sum = local_md5sum($files[$i]); 207 if ($instance_md5sum eq $md5sum) { 208 push @{ $good_instances{$instance_site} }, $i; 209 $is_good = 1; 210 $Ngood++; 211 } 212 else { 213 push @{ $bad_instances{$instance_site} }, $i; 214 } 141 215 } 142 216 else { 143 $existance[$i] = 0; 144 $md5sums[$i] = 'NON-EXISTANT'; 145 $md5sum_uniq{$md5sums[$i]} = 1; 146 } 147 ($diskhosts[$i],$diskvols[$i]) = parse_volume($files[$i]); 148 $validation[$i] = sprintf("% 3d %32s %s %d %s %d", 149 $existance[$i], 150 $md5sums[$i], 151 $files[$i], 152 $md5sums[$i] eq $md5sum, 153 $diskhosts[$i],$diskvols[$i] 154 ); 155 156 # Pre-parse decisions 157 if ($md5sums[$i] ne $md5sum) { 158 $quality[$i] = 0; 159 } 160 elsif ($existance[$i] == 0) { 161 $quality[$i] = 0; 162 } 163 elsif (is_backup_volume($diskhosts[$i])) { 164 $quality[$i] = 1; 165 $quality_mask = $quality_mask | 1; 166 $Ngood++; 167 } 168 elsif ($diskhosts[$i] eq $host_mapping{$class_id}) { 169 $quality[$i] = 1; 170 $quality_mask = $quality_mask | 2; 171 $Ngood++; 172 } 173 else { 174 $quality[$i] = 2; 175 $Ngood++; 176 } 177 } 178 179 print "\n$key $data_state $md5sum $hostname $class_id\n"; 180 if (1) { 181 182 print 183 "object id: ", @$stat[0], "\n", 184 "key: ", @$stat[1], "\n"; 185 print 186 "epoch: ", @$stat[4], "\n"; 187 print 188 "md5sum count: ", scalar(keys %md5sum_uniq), "\n"; 189 print 190 "requested instances: ", $user_copies, "\n", 191 "available instances: ", @$stat[6], "\n", 192 "existing instances: ", $existing_copies, "\n", 193 "total instances: ", @$stat[7], "\n", 194 "instance location:\n", " " x 4; 195 print 196 join("\n" . " " x 4, @validation), "\n"; 197 } 217 $instance_exists = 0; 218 $instance_md5sum = 'NON-EXISTANT'; 219 push @{ $bad_instances{$instance_site} }, $i; 220 } 221 222 $validation[$i] = sprintf(" % 3d %d %32s %s %s %s\n", 223 $instance_exists,$is_good,$instance_md5sum, 224 $files[$i],$instance_host,$instance_volume); 225 } 226 227 $timer = time() - $timer_start; 228 # object_id ext_id epoch available existing total timer 229 printf("%s %s %s %d %d %d %d\n",@$stat[0],@$stat[1],@$stat[4],@$stat[6],$existing_copies,@$stat[7],$timer); 230 # instance_exists is_good instance_md5sum file instance_host instance_volume 231 my $val_string = join('',@validation); 232 print "$val_string"; 198 233 199 234 200 235 # Decide what to do 236 237 ## This block attempts to find an out-of-nebulous instance with a good md5sum, 238 ## and substitutes it for the 0-th listed instance. This gives us one good 239 ## copy to work with. 201 240 if ($Ngood == 0) { 202 241 # DO something to attempt to fix this. … … 220 259 } 221 260 else { 222 $quality[0] = 1;223 261 print "cp $good_file $files[0]\n"; 224 system("cp $good_file $files[0]"); 262 if ($do_ops) { 263 system("cp $good_file $files[0]"); 264 } 225 265 } 226 266 # Begin my best validation thought … … 234 274 235 275 $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. 236 } 237 238 if ($quality[0] == 0) { # The first instance is bad. 239 # But since we're here, and not up there, there must be at least one good copy. Find it. 240 for (my $i = 0; $i <= $#md5sums; $i++) { 241 if ($md5sums[$i] eq $md5sum) { # Found it. 242 print "cp $files[$i] $files[0]\n"; 243 system("cp $files[$i] $files[0]"); 244 # Begin my best validation thought 245 { 246 my $tmpmd5 = local_md5sum($files[0]); 247 if ($tmpmd5 ne $md5sum) { 248 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 249 } 276 ## We've done work here, so we can't do a cull this iteration. 277 if ($do_cull == 1) { $do_cull = -1; } 278 } 279 280 ## We have more than zero bad copies. We may cull some of these in the future, but we should try to 281 ## leave everything in the best state possible. 282 printf(">> %d %d\n",$Ngood, $#files + 1); 283 if ($Ngood != $#files + 1) { 284 my $good_copy; 285 my $good_copy_index; 286 foreach my $site_key (keys %good_instances) { 287 if ($#{ $good_instances{$site_key} } != -1) { 288 $good_copy_index = $good_instances{$site_key}[0]; 289 $good_copy = $files[$good_copy_index]; 290 last; 291 } 292 } 293 printf(">> GOOD: $good_copy\n"); 294 foreach my $site_key (keys %bad_instances) { 295 foreach my $bad_copy_index (@{ $bad_instances{$site_key} }) { 296 print "cp $good_copy $files[$bad_copy_index]\n"; 297 if ($do_ops) { 298 system("cp $good_copy $files[$bad_copy_index]"); 250 299 } 251 # End my best validation thought. 252 last; # We're done here now. 253 } 254 } 255 } 256 257 if ($Ngood == 1) { # We have only one version 258 if ($quality_mask & 1) { # ANd it's on a backup volume 259 print "neb-replicate $key\n"; 260 if ($do_ops) { 261 $neb->replicate($key) or die "failed to replicate the single valid copy"; 262 if ($@) { die $@; } 263 264 } 265 266 } 267 else { # And it's not, so put one there 268 my $rep_vol = get_random_backup_volume(); 300 my $tmpmd5 = local_md5sum($files[$bad_copy_index]); 301 if ($tmpmd5 ne $md5sum) { 302 ## This isn't super critical, so we don't need to die here. 303 warn "Post-repair md5sum does not match! $tmpmd5 != $md5sum: $files[$bad_copy_index]"; 304 } 305 else { 306 ## success 307 push @{ $good_instances{$site_key} }, $bad_copy_index; 308 } 309 } 310 } 311 ## We've done work here, so we can't do a cull this iteration. 312 if ($do_cull == 1) { $do_cull = -1; } 313 } 314 315 ## We can now attempt to make replicated copies to the sites that require additional copies. 316 foreach my $site_key (keys %requirement_map) { 317 my $have_instances = $#{ $good_instances{$site_key} } + 1; 318 print "## $site_key $have_instances $requirement_map{$site_key}\n"; 319 if ($#{ $good_instances{$site_key} } + 1 < $requirement_map{$site_key}) { 320 my $rep_vol = get_random_site_volume($site_key); 269 321 print "neb-replicate --volume $rep_vol $key\n"; 270 322 if ($do_ops) { … … 277 329 @$uris = map {URI->new($_)->file if $_} @$uris; 278 330 my $tmpmd5 = local_md5sum(${ $uris }[0]); 331 332 my $validation_str = sprintf("% 3d %d %32s %s %s %s", 333 -1,-1,$tmpmd5, 334 ${ $uris }[0],"repl",$rep_vol); 335 print 336 join("\n" . " " x 4, $validation_str), "\n"; 337 279 338 if ($tmpmd5 ne $md5sum) { 280 339 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; … … 282 341 # End my best validation thought. 283 342 } 284 } 285 } 286 else { # N >= 2 287 if (!($quality_mask & 1)) { # no backup copy 288 my $rep_vol = get_random_backup_volume(); 289 print "neb-replicate --volume $rep_vol $key\n"; 290 if ($do_ops) { 291 $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; 292 if ($@) { die $@; } 293 294 # Begin my best validation thought 295 system("sync") == 0 or die "Couldn't sync?"; 296 my $uris = $neb->find_instances($key,$rep_vol); 297 @$uris = map {URI->new($_)->file if $_} @$uris; 298 my $tmpmd5 = local_md5sum(${ $uris }[0]); 299 if ($tmpmd5 ne $md5sum) { 300 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 301 } 302 # End my best validation thought. 303 304 } 305 } 306 if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid. 307 for (my $i = 0; $i <= $#files; $i++) { 308 if ($quality[$i] == 2) { 309 $quality[$i] = 1; 310 last; 311 } 312 } 313 } 314 for (my $i = 0; $i <= $#files; $i++) { 315 # print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n"; 316 if ($quality[$i] != 1) { 317 if ($existance[$i] == 0) { # This disk file doesn't exist. 318 system("touch $files[$i]"); 319 } 320 my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i]; 321 print "neb-cull --volume $cull_vol $key\n"; 322 if ($do_ops) { 323 # The tilde here is to force hard volumes. Don't touch it. 324 $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance"; 325 if ($@) { die "$@"; } 326 } 327 } 328 } 329 } 330 # exit(0); 331 } 343 ## We've done work here, so we can't do a cull this iteration. 344 if ($do_cull == 1) { $do_cull = -1; } 345 } 346 } 347 348 ## Do culls if that's what we were going to do. 349 if ($do_cull == -1) { 350 die "Cull option passed, but files were modified in the scan/repair/replicate phase. Not running cull!\n"; 351 } 352 elsif ($do_cull == 1) { 353 ## At this point, we should have no files in the bad_instances lists, because we've repaired them. 354 foreach my $site_key (keys %good_instances) { 355 if ($#{ $good_instances{$site_key} } + 1 > $requirement_map{$site_key}) { 356 for ($i = $requirement_map{$site_key}; $i <= $#{ $good_instances{$site_key} }; $i++) { 357 my $cull_index = ${ $good_instances{$site_key} }[$i]; 358 my ($instance_host,$instance_volume) = parse_volume($files[$cull_index]); 359 print "neb-cull --volume $instance_volume $key\n"; 360 if ($do_ops) { 361 # The tilde here is to force hard volumes. Don't touch it. 362 # Also: the 2 is a "minimum number of copies" restriction. Let's not be crazy here. 363 $neb->cull($key,"~${instance_volume}",2) or die "failed to cull a superfluous instance"; 364 if ($@) { die "$@"; } 365 } 366 } # End loop over extra instances 367 } # End check for sites with extra instances 368 } # End loop over sites. 369 } # End cull 370 371 } ## End loop over imfiles. 332 372 333 373 sub local_md5sum { … … 350 390 my $filename = shift(@_); 351 391 my $full_volume = (split /\//, $filename)[2]; 352 my ($hostname,$vol_index) = split /\./, $full_volume; # /; 353 return($hostname,$vol_index); 354 } 355 356 sub is_backup_volume { 357 my $hostname = shift(@_); 358 if (exists($backup_hosts{$hostname})) { 359 return(1); 360 } 361 return(0); 362 } 363 364 sub get_random_backup_volume { 365 my $NN = scalar keys %backup_destinations; 366 my $backup_host = (keys %backup_destinations)[int(rand($NN))]; 367 my $backup_vol = int(rand($backup_Nvols)); 368 369 return("${backup_host}.${backup_vol}"); 392 my ($hostname,undef) = split /\./, $full_volume; # /; 393 return($hostname,$full_volume); 394 } 395 396 sub get_random_site_volume { 397 my $site_key = shift(@_); 398 my $NN = scalar @{ $volume_lists{$site_key} }; 399 my $backup_volume = ${ $volume_lists{$site_key} }[int(rand($NN))]; 400 return($backup_volume); 370 401 } 371 402
Note:
See TracChangeset
for help on using the changeset viewer.
