Changeset 39900 for trunk/ippScripts/scripts/permcheck.pl
- Timestamp:
- Dec 20, 2016, 4:09:22 PM (10 years ago)
- File:
-
- 1 edited
-
trunk/ippScripts/scripts/permcheck.pl (modified) (9 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/permcheck.pl
r37833 r39900 25 25 my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); 26 26 27 my ($server,$dbname,$stage,$stage_id );27 my ($server,$dbname,$stage,$stage_id,$do_cull,$save_log); 28 28 29 29 $server = $ENV{'NEB_SERVER'} unless $server; … … 34 34 'stage=s' => \$stage, 35 35 'stage_id|x=s' => \$stage_id, 36 'save_log' => \$save_log, 37 'cull' => \$do_cull, 36 38 ) || pod2usage( 2 ); 39 40 unless(defined($do_cull)) { 41 $do_cull = 0; 42 } 37 43 38 44 # Option parsing … … 47 53 unless defined $stage_id; 48 54 55 if ($save_log) { 56 my $time = time(); 57 my $logDest = "neb://any/perm_check/${stage}/${stage}_${stage_id}.${time}"; 58 my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n"; 59 $ipprc->redirect_output($logDest) or die "Could not redirect output to logfile ${logDest}\n"; 60 } 61 62 49 63 # Global options: 50 64 ## Define the configuration. Ideally, this would be retrieved from the nebulous 65 ## database, but there are some issues with that (such as grouping the b nodes 66 ## into a less restrictive "offsite" location). 51 67 my $do_ops = 1; 52 my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1, 53 'ippb02' => 1, 'ippb03' => 1, 54 'ippb04' => 1, 'ippb05' => 1, 55 'ippb06' => 1 56 ); 57 my %backup_destinations = (#'ippb04' => 1, 58 #'ippb05' => 1, 59 'ippb06' => 1 60 ); 61 62 if ($stage eq 'warp') { # These things are destined to have their only copy on the stsciXX nodes. 63 %backup_hosts = ('stsci00' => 1, 'stsci01' => 1, 'stsci02' => 1, 64 #'stsci03' => 1, 65 'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1, 66 'stsci07' => 1, 'stsci08' => 1, 'stsci09' => 1, 67 'stsci10' => 1, 'stsci11' => 1, 'stsci12' => 1, 68 'stsci13' => 1, 'stsci14' => 1, 'stsci15' => 1, 'stsci16' => 1, 69 'stsci17' => 1, 'stsci18' => 1, 'stsci19' => 1); 70 %backup_destinations = %backup_hosts; 71 } 72 73 my $backup_Nvols = 3; 74 75 my $ipprc; 76 if ($dbname eq 'gpc1') { 77 $ipprc = PS::IPP::Config->new( "GPC1" ); 78 } 79 else { 80 die "Unknown camera to use."; 81 } 68 my $i; 82 69 83 70 # Set up nebulous db interface … … 88 75 unless defined $neb; 89 76 77 ## This new implementation is somewhat messy, but is more general and adaptable. 78 ## First, we set up a requirement mapping, explaining where we want copies, and 79 ## how many copies we want at each site. 80 my %requirement_map = (); 81 $requirement_map{ITC} = 1; 82 $requirement_map{OFFSITE} = 1; 83 $requirement_map{MRTCB} = 0; 84 85 ## Second, construct a list of volumes, mapped to their site location, using the 86 ## same site locations as in the requirement map. 87 my %volume_map = (); 88 for ($i = 4; $i <= 21; $i++) { 89 my $vol = sprintf("ipp%03d.0",$i); 90 $volume_map{$vol} = 'MRTCB'; 91 } 92 for ($i = 23; $i <= 32; $i++) { 93 my $vol = sprintf("ipp%03d.0",$i); 94 $volume_map{$vol} = 'MRTCB'; 95 } 96 for ($i = 54; $i <= 97; $i++) { 97 my $vol = sprintf("ipp%03d.0",$i); 98 $volume_map{$vol} = 'MRTCB'; 99 } 100 for ($i = 100; $i <= 104; $i++) { 101 my $vol = sprintf("ipp%03d.0",$i); 102 $volume_map{$vol} = 'MRTCB'; 103 $vol = sprintf("ipp%03d.1",$i); 104 $volume_map{$vol} = 'MRTCB'; 105 } 106 for ($i = 105; $i <= 117; $i++) { 107 # if ($i == 115) { next; } 108 my $vol = sprintf("ipp%03d.0",$i); 109 $volume_map{$vol} = 'ITC'; 110 $vol = sprintf("ipp%03d.1",$i); 111 $volume_map{$vol} = 'ITC'; 112 } 113 for ($i = 0; $i <= 15; $i++) { 114 if ($i == 6) { next; } # This isn't "offsite", it's with the rest of the maui cluster. 115 if ($i == 9) { next; } # Not online 116 117 my $vol = sprintf("ippb%02d.0",$i); 118 $volume_map{$vol} = 'OFFSITE'; 119 $vol = sprintf("ippb%02d.1",$i); 120 $volume_map{$vol} = 'OFFSITE'; 121 $vol = sprintf("ippb%02d.2",$i); 122 $volume_map{$vol} = 'OFFSITE'; 123 } 124 125 ## Next, get disk space values, and check which hosts are listed as available. 126 my %acceptable_volume = (); 127 my $mounts = $neb->mounts(); 128 foreach my $vol_row (@$mounts) { 129 my ($mount_point, $total, $used, $vol_id, $name, $host, $path, $allocate, $available, $xattr) = @{ $vol_row }; 130 if (($allocate == 1)&&($available == 1)&&( $used / $total < 0.98)) { 131 $acceptable_volume{$name} = 1; 132 } 133 else { 134 print "## $name $allocate $available $used $total\n"; 135 $acceptable_volume{$name} = 0; 136 } 137 if ($name =~ /ippb05/) { 138 $acceptable_volume{$name} = 0; 139 } 140 # if (($name eq 'ipp106.0')||($name eq 'ipp106.1')) { 141 # $acceptable_volume{$name} = 0; 142 # } 143 } 144 145 ## Finally, generate lists containing which volumes are located at which site. 146 ## This allows us to randomly select a volume from the list for the site that 147 ## we plan on replicating to. 148 my %volume_lists = (); 149 foreach my $vol_key (keys %requirement_map) { 150 @{ $volume_lists{$vol_key} } = grep { $acceptable_volume{$_} == 1 } ( 151 grep { $volume_map{$_} eq $vol_key } (keys %volume_map) 152 ); 153 print "$vol_key " . join(' ', @{ $volume_lists{$vol_key} }) . "\n"; 154 155 if ($#{ $volume_lists{$vol_key} } == -1) { 156 die "No acceptable volume found for site $vol_key!\n"; 157 } 158 } 159 90 160 # Pull data from the gpc1 database 91 161 my $verbose = 0; 92 162 my $mdcParser = PS::IPP::Metadata::Config->new; 93 163 94 my $files; 164 # Not technically imfiles, but the nebulous check block already uses files as a variable. 165 my $imfiles; 95 166 if ($stage eq 'camera') { 96 167 my $cmd = "$camtool -processedexp -cam_id $stage_id -dbname $dbname"; … … 101 172 &my_die("Unable to perform stagetool: $error_code", $stage_id); 102 173 } 103 $ files = $mdcParser->parse_list(join "", @$stdout_buf) or174 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 104 175 &my_die("Unable to parse metadata from stagetool", $stage_id); 105 176 } … … 112 183 &my_die("Unable to perform stagetool: $error_code", $stage_id); 113 184 } 114 $ files = $mdcParser->parse_list(join "", @$stdout_buf) or185 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 115 186 &my_die("Unable to parse metadata from stagetool", $stage_id); 116 187 } … … 123 194 &my_die("Unable to perform stagetool: $error_code", $stage_id); 124 195 } 125 $ files = $mdcParser->parse_list(join "", @$stdout_buf) or196 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 126 197 &my_die("Unable to parse metadata from stagetool", $stage_id); 127 198 } … … 134 205 &my_die("Unable to perform stagetool: $error_code", $stage_id); 135 206 } 136 $ files = $mdcParser->parse_list(join "", @$stdout_buf) or207 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 137 208 &my_die("Unable to parse metadata from stagetool", $stage_id); 138 209 } 139 140 my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'], 210 elsif ($stage eq 'ff') { 211 my $cmd = "$fftool -result --ff_id $stage_id -dbname $dbname"; 212 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 213 run(command => $cmd, verbose => 0); 214 unless ($success) { 215 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 216 &my_die("Unable to perform stagetool: $error_code", $stage_id); 217 } 218 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 219 &my_die("Unable to parse metadata from stagetool", $stage_id); 220 } 221 222 223 my $timer_start = time(); 224 my $timer = time(); 225 my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK','PSPHOT.BACKMDL','PSPHOT.PSF.RAW.SAVE'], 141 226 'warp' => ['PSWARP.OUTPUT','PSWARP.OUTPUT.MASK','PSWARP.OUTPUT.VARIANCE'], 142 227 'stack' => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP', 143 228 'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'], 144 'skycal' => ['PSASTRO.OUTPUT.CMF']); 145 146 147 foreach my $entry (@$files) { 229 'skycal' => ['PSASTRO.OUTPUT.CMF','PSPHOT.OUTPUT.CFF'], 230 'ff' => ['PSPHOT.OUT.CMF.MEF','PSPHOT.OUTPUT.CFF','PSPHOT.FULLFORCE.OUTPUT']); 231 232 my $timer_start = time(); 233 my $timer = time(); 234 235 print "## permcheck.pl: $stage $stage_id $dbname $do_cull $do_ops $timer_start\n"; 236 foreach my $entry (@$imfiles) { 148 237 my $path_base = $entry->{path_base}; 149 238 my $data_state = $entry->{state}; 150 239 my $hostname = $entry->{hostname}; 151 240 my $quality = $entry->{quality}; 241 242 $timer = $time() - $timer_start; 243 print "# $path_base $data_state $hostname $quality T: $timer\n"; 152 244 if ($quality != 0) { next; } 153 print "# $path_base $data_state $hostname $quality\n"; 245 246 my @keys = (); 154 247 foreach my $product (@{ $components{$stage} }) { 155 my @keys = (); 156 if (($stage eq 'camera')&&($product eq 'PSASTRO.OUTPUT.MASK')) { 157 my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06', 158 'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17', 159 'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27', 160 'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37', 161 'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47', 162 'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57', 163 'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67', 164 'XY71','XY72','XY73','XY74','XY75','XY76'); 165 foreach my $ota (@otas) { 166 push @keys, $ipprc->filename($product,$path_base,$ota); 167 } 168 } 248 if ($stage eq 'camera') { 249 if ($product eq 'PSASTRO.OUTPUT.MASK') { 250 my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06', 251 'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17', 252 'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27', 253 'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37', 254 'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47', 255 'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57', 256 'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67', 257 'XY71','XY72','XY73','XY74','XY75','XY76'); 258 foreach my $ota (@otas) { 259 push @keys, $ipprc->filename($product,$path_base,$ota); 260 } 261 } 262 elsif (($product eq 'PSPHOT.BACKMDL')||($product eq 'PSPHOT.PSF.RAW.SAVE')) { 263 my $chip_id = $entry->{chip_id}; 264 my $chip_cmd = "$chiptool -processedimfile -chip_id $chip_id -dbname $dbname"; 265 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 266 run(command => $chip_cmd, verbose => 0); 267 unless ($success) { 268 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 269 &my_die("Unable to perform stagetool: $error_code", $stage_id); 270 } 271 my $chip_$files = $mdcParser->parse_list(join "", @$stdout_buf) or 272 &my_die("Unable to parse metadata from stagetool", $stage_id); 273 274 foreach my $chip_entry (@$chip_files) { 275 my $chip_path_base = $chip_entry->{path_base}; 276 my $class_id = $chip_entry->{class_id}; 277 push @keys, $ipprc->filename($product,$chip_path_base,$class_id); 278 } 279 } 280 } 281 elsif (($stage eq 'ff')&&($product eq 'PSPHOT.FULLFORCE.OUTPUT')) { 282 my $ffsum_cmd = "$fftool -summary -ff_id $stage_id -dbname $dbname"; 283 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 284 run(command => $ffsum_cmd, verbose => 0); 285 unless ($success) { 286 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 287 &my_die("Unable to perform stagetool: $error_code", $stage_id); 288 } 289 my $chip_$files = $mdcParser->parse_list(join "", @$stdout_buf) or 290 &my_die("Unable to parse metadata from stagetool", $stage_id); 291 292 foreach my $chip_entry (@$chip_files) { 293 my $chip_path_base = $chip_entry->{path_base}; 294 my $class_id = $chip_entry->{class_id}; 295 push @keys, $ipprc->filename($product,$chip_path_base,); 296 } 297 } 169 298 else { 170 299 push @keys, $ipprc->filename($product,$path_base); 171 300 } 172 301 # Do validation 302 } 303 304 $timer = time() - $timer_start; 305 306 printf("# Identified %d : %s\n",$#keys + 1, $timer); 307 foreach my $key (@keys) { 308 # neb-stat level handling 309 my $stat = $neb->stat($key); 310 die "nebulous key: $key not found" unless $stat; 311 my $instances; 312 my $md5sum = ''; 313 314 # This needs to be in an eval, because although we expect things to exist, they 315 # may not. This is a fatal error in rawcheck, but need not be here. 316 eval { 317 $instances = $neb->find_instances($key, 'any'); 318 }; 319 unless (defined($instances)) { print "## skipping due to zero instances\n"; next; } 320 die "no instances found" unless $instances; 173 321 174 foreach my $key (@keys) { 175 # neb-stat level handling 176 my $stat = $neb->stat($key); 177 die "nebulous key: $key not found" unless $stat; 178 my $instances; 179 eval { 180 $instances = $neb->find_instances($key, 'any'); 181 }; 182 unless (defined($instances)) { print "## skipping due to zero instances\n"; next; } 183 die "no instances found" unless $instances; 322 my $Ngood; 323 my %good_instances = (); 324 my %bad_instances = (); 325 326 my @files = map {URI->new($_)->file if $_} @$instances; 327 my @validation = (); 328 329 for (my $i = 0; $i <= $#files; $i++) { 330 my ($instance_exists,$instance_md5sum,$instance_host,$instance_volume,$instance_site, $is_good); 331 ($instance_host,$instance_volume) = parse_volume($files[$i]); 332 $instance_site = $volume_map{$instance_volume}; 333 $is_good = 0; 334 335 if (-e $files[$i]) { 336 $instance_exists = 1; 337 $existing_copies++; 338 $instance_md5sum = local_md5sum($files[$i]); 339 340 # This is bad, but I don't know what the right solution is. We don't have 341 # the md5sum a priori. I think this is also the only major change needed 342 # from rawcheck.pl 343 if (($md5sum eq '')&&($instance_md5sum ne 'd41d8cd98f00b204e9800998ecf8427e')) { 344 $md5sum = $instance_md5sum; 345 } 346 347 if ($instance_md5sum eq $md5sum) { 348 push @{ $good_instance{$instance_site} }, $i; 349 $is_good = 1; 350 $Ngood++; 351 } 352 else { 353 push @{ $bad_instances{$instance_site} }, $i; 354 } 355 } 356 else { 357 $instance_exists = 0; 358 $instance_md5sum = 'NON-EXISTANT'; 359 push @{ $bad_instances{$instance_site} }, $i; 360 } 184 361 185 my $user_copies; 186 eval { 187 $user_copies = $neb->getxattr($key, "user.copies"); 188 }; 189 unless(defined($user_copies)) { 190 $user_copies = 1; 191 } 192 193 my $md5sum; 194 my @validation; 195 my %md5sum_uniq; 196 my $existing_copies = 0; 197 198 my @existance; 199 my @md5sums; 200 my @diskfiles = map {URI->new($_)->file if $_} @$instances; 201 my @diskvols; 202 my @diskhosts; 203 my @quality; 204 my $Ngood = 0; 205 my $quality_mask = 0; 206 207 for (my $i = 0; $i <= $#diskfiles; $i++) { 208 if (-e $diskfiles[$i]) { 209 $existance[$i] = 1; 210 $existing_copies++; 211 $md5sums[$i] = local_md5sum($diskfiles[$i]); 212 $md5sum_uniq{$md5sums[$i]} = 1; 213 } 214 else { 215 $existance[$i] = 0; 216 $md5sums[$i] = 'NON-EXISTANT'; 217 $md5sum_uniq{$md5sums[$i]} = 1; 218 } 219 ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]); 220 $validation[$i] = sprintf("% 3d %32s %s %s %d", 221 $existance[$i], 222 $md5sums[$i], 223 $diskfiles[$i], 224 $diskhosts[$i],$diskvols[$i] 225 ); 226 227 # Pre-parse decisions 228 if ($existance[$i] == 0) { 229 $quality[$i] = 0; 230 } 231 elsif (is_backup_volume($diskhosts[$i])) { 232 $quality[$i] = 1; 233 $quality_mask = $quality_mask | 1; 234 $Ngood++; 235 } 236 else { 237 $quality[$i] = 2; 238 $Ngood++; 239 } 240 } 241 if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth. 242 die "There are multiple md5sum values for $key"; 243 } 244 $md5sum = (keys(%md5sum_uniq))[0]; 245 246 print "\n$key $data_state $md5sum $hostname\n"; 247 if (1) { 248 249 print 250 "object id: ", @$stat[0], "\n", 251 "key: ", @$stat[1], "\n"; 252 print 253 "epoch: ", @$stat[4], "\n"; 254 print 255 "md5sum count: ", scalar(keys %md5sum_uniq), "\n"; 256 print 257 "requested instances: ", $user_copies, "\n", 258 "available instances: ", @$stat[6], "\n", 259 "existing instances: ", $existing_copies, "\n", 260 "total instances: ", @$stat[7], "\n", 261 "instance location:\n", " " x 4; 262 print 263 join("\n" . " " x 4, @validation), "\n"; 264 } 265 362 $validation[$i] = sprintf(" % 3d %d %32s %s %s %s\n", 363 $instance_exists,$is_good,$instance_md5sum, 364 $files[$i],$instance_host,$instance_volume); 365 } 366 367 $time = time() - $timer_start; 368 # object_id ext_id epoch available existing total timer 369 printf("%s %s %s %d %d %d %d\n",@$stat[0],@$stat[1],@$stat[4],@$stat[6],$existing_copies,@$stat[7],$timer); 370 # instance_exists is_good instance_md5sum file instance_host instance_volume 371 my $val_string = join('',@validation); 372 print "$val_string"; 373 266 374 # Decide what to do 267 if ($Ngood == 0) { 268 # DO something to attempt to fix this. 269 my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; 270 271 open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); 272 my $good_file = ''; 273 while (<DD>) { 274 $_ =~ s/^\s+//; 275 my ($z,undef,$ff) = split /\s+/; 276 if (($ff)&&(-e $ff)) { 277 my $md_response = `md5sum $ff`; 278 if ($md_response =~ /$md5sum/) { 279 $good_file = (split /\s+/,$md_response)[1]; 280 } 375 376 ## This block attempts to find an out-of-nebulous instance with a good md5sum, 377 ## and substitutes it for the 0-th listed instance. This gives us one good 378 ## copy to work with. 379 if ($Ngood == 0) { 380 # DO something to attempt to fix this. 381 my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; 382 383 open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); 384 my $good_file = ''; 385 while (<DD>) { 386 $_ =~ s/^\s+//; 387 my ($z,undef,$ff) = split /\s+/; 388 if (($ff)&&(-e $ff)) { 389 my $md_response = `md5sum $ff`; 390 if ($md_response =~ /$md5sum/) { 391 $good_file = (split /\s+/,$md_response)[1]; 281 392 } 282 393 } 283 close(DD); 284 if ($good_file eq '') { 285 die "No valid instance of key: $key"; 286 } 287 else { 288 $quality[0] = 1; 289 print "cp $good_file $diskfiles[0]\n"; 290 vsystem("cp $good_file $diskfiles[0]"); 291 } 292 # Begin my best validation thought 293 { 294 my $tmpmd5 = local_md5sum($diskfiles[0]); 394 } 395 close(DD); 396 if ($good_file eq '') { 397 die "No valid instance of key: $key"; 398 } 399 else { 400 print "cp $good_file $files[0]\n"; 401 if ($do_ops) { 402 system("cp $good_file $files[0]"); 403 } 404 } 405 # Begin my best validation thought 406 { 407 my $tmpmd5 = local_md5sum($files[0]); 408 if ($tmpmd5 ne $md5sum) { 409 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 410 } 411 } 412 # End my best validation thought. 413 414 $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. 415 ## We've done work here, so we can't do a cull this iteration. 416 if ($do_cull == 1) { $do_cull = -1; } 417 } 418 ## We have more than zero bad copies. We may cull some of these in the future, but we should try to 419 ## leave everything in the best state possible. 420 printf(">> %d %d\n",$Ngood, $#files + 1); 421 if ($Ngood != $#files + 1) { 422 my $good_copy; 423 my $good_copy_index; 424 foreach my $site_key (keys %good_instances) { 425 if ($#{ $good_instances{$site_key} } != -1) { 426 $good_copy_index = $good_instances{$site_key}[0]; 427 $good_copy = $files[$good_copy_index]; 428 last; 429 } 430 } 431 printf(">> GOOD: $good_copy\n"); 432 foreach my $site_key (keys %bad_instances) { 433 foreach my $bad_copy_index (@{ $bad_instances{$site_key} }) { 434 print "cp $good_copy $files[$bad_copy_index]\n"; 435 if ($do_ops) { 436 system("cp $good_copy $files[$bad_copy_index]"); 437 } 438 my $tmpmd5 = local_md5sum($files[$bad_copy_index]); 439 if ($tmpmd5 ne $md5sum) { 440 ## This isn't super critical, so we don't need to die here. 441 warn "Post-repair md5sum does not match! $tmpmd5 != $md5sum: $files[$bad_copy_index]"; 442 } 443 else { 444 ## success 445 push @{ $good_instances{$site_key} }, $bad_copy_index; 446 } 447 } 448 } 449 ## We've done work here, so we can't do a cull this iteration. 450 if ($do_cull == 1) { $do_cull = -1; } 451 } 452 453 ## We can now attempt to make replicated copies to the sites that require additional copies. 454 foreach my $site_key (keys %requirement_map) { 455 my $have_instances = $#{ $good_instances{$site_key} } + 1; 456 print "## $site_key $have_instances $requirement_map{$site_key}\n"; 457 if ($#{ $good_instances{$site_key} } + 1 < $requirement_map{$site_key}) { 458 my $rep_vol = get_random_site_volume($site_key); 459 print "neb-replicate --volume $rep_vol $key\n"; 460 if ($do_ops) { 461 $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; 462 if ($@) { die $@; } 463 464 # Begin my best validation thought 465 system("sync") == 0 or die "Couldn't sync?"; 466 my $uris = $neb->find_instances($key,$rep_vol); 467 @$uris = map {URI->new($_)->file if $_} @$uris; 468 my $tmpmd5 = local_md5sum(${ $uris }[0]); 469 470 my $validation_str = sprintf("% 3d %d %32s %s %s %s", 471 -1,-1,$tmpmd5, 472 ${ $uris }[0],"repl",$rep_vol); 473 print 474 join("\n" . " " x 4, $validation_str), "\n"; 475 295 476 if ($tmpmd5 ne $md5sum) { 296 477 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 297 478 } 298 } 299 # End my best validation thought. 300 301 $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. 302 } 303 304 if ($quality[0] == 0) { # The first instance is bad. 305 # But since we're here, and not up there, there must be at least one good copy. Find it. 306 for (my $i = 0; $i <= $#md5sums; $i++) { 307 if ($md5sums[$i] eq $md5sum) { # Found it. 308 print "cp $diskfiles[$i] $diskfiles[0]\n"; 309 vsystem("cp $diskfiles[$i] $diskfiles[0]"); 310 # Begin my best validation thought 311 { 312 my $tmpmd5 = local_md5sum($diskfiles[0]); 313 if ($tmpmd5 ne $md5sum) { 314 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 315 } 316 } 317 # End my best validation thought. 318 last; # We're done here now. 319 } 320 } 321 } 322 323 if ($Ngood == 1) { # We have only one version 324 if ($quality_mask & 1) { # ANd it's on a backup volume 325 if ($user_copies > 1) { # And we want more than one copy. 326 print "neb-replicate $key\n"; 327 if ($do_ops) { 328 $neb->replicate($key) or die "failed to replicate the single valid copy"; 329 if ($@) { die $@; } 330 } 331 } 332 } 333 else { # And it's not, so put one there 334 my $rep_vol = get_random_backup_volume(); 335 print "neb-replicate --volume $rep_vol $key\n"; 336 if ($do_ops) { 337 $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; 338 if ($@) { die $@; } 339 340 # Begin my best validation thought 341 vsystem("sync") == 0 or die "Couldn't sync?"; 342 my $uris = $neb->find_instances($key,$rep_vol); 343 @$uris = map {URI->new($_)->file if $_} @$uris; 344 my $tmpmd5 = local_md5sum(${ $uris }[0]); 345 if ($tmpmd5 ne $md5sum) { 346 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 347 } 348 # End my best validation thought. 349 } 350 } 351 } 352 else { # N >= 2 353 if (!($quality_mask & 1)) { # no backup copy 354 my $rep_vol = get_random_backup_volume(); 355 print "neb-replicate --volume $rep_vol $key\n"; 356 if ($do_ops) { 357 $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; 358 if ($@) { die $@; } 359 360 # Begin my best validation thought 361 vsystem("sync") == 0 or die "Couldn't sync?"; 362 my $uris = $neb->find_instances($key,$rep_vol); 363 @$uris = map {URI->new($_)->file if $_} @$uris; 364 my $tmpmd5 = local_md5sum(${ $uris }[0]); 365 if ($tmpmd5 ne $md5sum) { 366 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 367 } 368 # End my best validation thought. 369 } 370 } 371 if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid. 372 for (my $i = 0; $i <= $#diskfiles; $i++) { 373 if ($quality[$i] == 2) { 374 $quality[$i] = 1; 375 last; 376 } 377 } 378 } 379 } 380 #XXX NEW DEBUG THIS: This should iterate over diskfiles that are not marked with a quality = 2, and cull them. 381 # For the N>=2, we set that for the primary, and 382 # for both cases, we do not have a diskfile entry for the newly replicated copy. 383 # Therefore, this should cull down to the correct number. 384 for (my $i = 0; $i <= $#diskfiles; $i++) { 385 # print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n"; 386 if ($quality[$i] != 1) { 387 if ($existance[$i] == 0) { # This disk file doesn't exist. 388 vsystem("touch $diskfiles[$i]"); 389 } 390 my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i]; 391 print "neb-cull --volume $cull_vol $key\n"; 392 if ($do_ops) { 393 # The tilde here is to force hard volumes. Don't touch it. 394 $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance"; 395 if ($@) { die "$@"; } 396 } 397 } 398 } 399 # } 400 } # end keys for this product 401 } # end product for this entry 402 } # end entry for this id 403 404 sub vsystem { 405 my $cmd = shift; 406 print "$cmd\n"; 407 if ($do_ops) { 408 system($cmd); 409 } 410 } 479 # End my best validation thought. 480 } 481 ## We've done work here, so we can't do a cull this iteration. 482 if ($do_cull == 1) { $do_cull = -1; } 483 } 484 } 485 486 ## Do culls if that's what we were going to do. 487 if ($do_cull == -1) { 488 die "Cull option passed, but files were modified in the scan/repair/replicate phase. Not running cull!\n"; 489 } 490 elsif ($do_cull == 1) { 491 ## At this point, we should have no files in the bad_instances lists, because we've repaired them. 492 foreach my $site_key (keys %good_instances) { 493 if ($#{ $good_instances{$site_key} } + 1 > $requirement_map{$site_key}) { 494 for ($i = $requirement_map{$site_key}; $i <= $#{ $good_instances{$site_key} }; $i++) { 495 my $cull_index = ${ $good_instances{$site_key} }[$i]; 496 my ($instance_host,$instance_volume) = parse_volume($files[$cull_index]); 497 print "neb-cull --volume $instance_volume $key\n"; 498 if ($do_ops) { 499 # The tilde here is to force hard volumes. Don't touch it. 500 # Also: the 2 is a "minimum number of copies" restriction. Let's not be crazy here. 501 $neb->cull($key,"~${instance_volume}",2) or die "failed to cull a superfluous instance"; 502 if ($@) { die "$@"; } 503 } 504 } # End loop over extra instances 505 } # End check for sites with extra instances 506 } # End loop over sites. 507 } # End cull 508 509 } ## End loop over imfiles. 510 411 511 412 512 sub local_md5sum { … … 425 525 } 426 526 427 428 429 527 sub parse_volume { 430 528 my $filename = shift(@_); 431 529 my $full_volume = (split /\//, $filename)[2]; 432 my ($hostname,$vol_index) = split /\./, $full_volume; # /; 433 return($hostname,$vol_index); 434 } 435 436 sub is_backup_volume { 437 my $hostname = shift(@_); 438 if (exists($backup_hosts{$hostname})) { 439 return(1); 440 } 441 return(0); 442 } 443 444 sub get_random_backup_volume { 445 my $NN = scalar keys %backup_destinations; 446 my $backup_host = (keys %backup_destinations)[int(rand($NN))]; 447 my $backup_vol = int(rand($backup_Nvols)); 448 449 return("${backup_host}.${backup_vol}"); 530 my ($hostname,undef) = split /\./, $full_volume; # /; 531 return($hostname,$full_volume); 532 } 533 534 sub get_random_site_volume { 535 my $site_key = shift(@_); 536 my $NN = scalar @{ $volume_lists{$site_key} }; 537 my $backup_volume = ${ $volume_lists{$site_key} }[int(rand($NN))]; 538 return($backup_volume); 450 539 } 451 540
Note:
See TracChangeset
for help on using the changeset viewer.
