Changeset 39957
- Timestamp:
- Jan 27, 2017, 4:51:55 PM (9 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
tags/ipp-20150312/ippScripts/scripts/permcheck.pl
- Property svn:mergeinfo set to
r37833 r39957 20 20 my $missing_tools = 0; 21 21 #my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); 22 my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); 22 23 my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); 23 24 my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); 24 25 my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); 25 26 my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); 26 27 my ($server,$dbname,$stage,$stage_id); 27 my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1); 28 my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); 29 30 my ($server,$dbname,$stage,$stage_id,$do_cull,$save_log); 28 31 29 32 $server = $ENV{'NEB_SERVER'} unless $server; … … 34 37 'stage=s' => \$stage, 35 38 'stage_id|x=s' => \$stage_id, 39 'save_log' => \$save_log, 40 'cull' => \$do_cull, 36 41 ) || pod2usage( 2 ); 42 43 unless(defined($do_cull)) { 44 $do_cull = 0; 45 } 37 46 38 47 # Option parsing … … 47 56 unless defined $stage_id; 48 57 58 my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n"; 59 60 if ($save_log) { 61 my $time = time(); 62 my $logDest = "neb://any/perm_check/${stage}/${stage}_${stage_id}"; 63 $ipprc->redirect_to_logfile($logDest) or die "Could not redirect output to logfile ${logDest}\n"; 64 } 65 66 49 67 # Global options: 50 68 ## Define the configuration. Ideally, this would be retrieved from the nebulous 69 ## database, but there are some issues with that (such as grouping the b nodes 70 ## into a less restrictive "offsite" location). 51 71 my $do_ops = 1; 52 my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1, 53 'ippb02' => 1, 'ippb03' => 1, 54 'ippb04' => 1, 'ippb05' => 1, 55 'ippb06' => 1 56 ); 57 my %backup_destinations = (#'ippb04' => 1, 58 #'ippb05' => 1, 59 'ippb06' => 1 60 ); 61 62 if ($stage eq 'warp') { # These things are destined to have their only copy on the stsciXX nodes. 63 %backup_hosts = ('stsci00' => 1, 'stsci01' => 1, 'stsci02' => 1, 64 #'stsci03' => 1, 65 'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1, 66 'stsci07' => 1, 'stsci08' => 1, 'stsci09' => 1, 67 'stsci10' => 1, 'stsci11' => 1, 'stsci12' => 1, 68 'stsci13' => 1, 'stsci14' => 1, 'stsci15' => 1, 'stsci16' => 1, 69 'stsci17' => 1, 'stsci18' => 1, 'stsci19' => 1); 70 %backup_destinations = %backup_hosts; 71 } 72 73 my $backup_Nvols = 3; 74 75 my $ipprc; 76 if ($dbname eq 'gpc1') { 77 $ipprc = PS::IPP::Config->new( "GPC1" ); 78 } 79 else { 80 die "Unknown camera to use."; 81 } 72 my $i; 82 73 83 74 # Set up nebulous db interface … … 88 79 unless defined $neb; 89 80 81 ## This new implementation is somewhat messy, but is more general and adaptable. 82 ## First, we set up a requirement mapping, explaining where we want copies, and 83 ## how many copies we want at each site. 84 my %requirement_map = (); 85 $requirement_map{ITC} = 1; 86 $requirement_map{OFFSITE} = 0; 87 $requirement_map{MRTCB} = 1; 88 89 ## Second, construct a list of volumes, mapped to their site location, using the 90 ## same site locations as in the requirement map. 91 my %volume_map = (); 92 for ($i = 4; $i <= 21; $i++) { 93 my $vol = sprintf("ipp%03d.0",$i); 94 $volume_map{$vol} = 'MRTCB'; 95 } 96 for ($i = 23; $i <= 32; $i++) { 97 my $vol = sprintf("ipp%03d.0",$i); 98 $volume_map{$vol} = 'MRTCB'; 99 } 100 for ($i = 54; $i <= 97; $i++) { 101 my $vol = sprintf("ipp%03d.0",$i); 102 $volume_map{$vol} = 'MRTCB'; 103 } 104 for ($i = 100; $i <= 104; $i++) { 105 my $vol = sprintf("ipp%03d.0",$i); 106 $volume_map{$vol} = 'MRTCB'; 107 $vol = sprintf("ipp%03d.1",$i); 108 $volume_map{$vol} = 'MRTCB'; 109 } 110 for ($i = 105; $i <= 117; $i++) { 111 # if ($i == 115) { next; } 112 my $vol = sprintf("ipp%03d.0",$i); 113 $volume_map{$vol} = 'ITC'; 114 $vol = sprintf("ipp%03d.1",$i); 115 $volume_map{$vol} = 'ITC'; 116 } 117 for ($i = 118; $i <= 122; $i++) { 118 my $vol = sprintf("ipp%03d.0",$i); 119 $volume_map{$vol} = 'MRTCB'; 120 $vol = sprintf("ipp%03d.1",$i); 121 $volume_map{$vol} = 'MRTCB'; 122 } 123 124 125 for ($i = 0; $i <= 15; $i++) { 126 my $loc = 'OFFSITE'; 127 if ($i == 6) { 128 $loc = 'MRTCB'; 129 } # This isn't "offsite", it's with the rest of the maui cluster. 130 if ($i == 9) { next; } # Not online 131 132 my $vol = sprintf("ippb%02d.0",$i); 133 $volume_map{$vol} = $loc; 134 $vol = sprintf("ippb%02d.1",$i); 135 $volume_map{$vol} = $loc; 136 if ($i <= 6) { 137 $vol = sprintf("ippb%02d.2",$i); 138 $volume_map{$vol} = $loc; 139 } 140 } 141 142 ## Next, get disk space values, and check which hosts are listed as available. 143 my %acceptable_volume = (); 144 my $mounts = $neb->mounts(); 145 foreach my $vol_row (@$mounts) { 146 my ($mount_point, $total, $used, $vol_id, $name, $host, $path, $allocate, $available, $xattr) = @{ $vol_row }; 147 if (($allocate == 1)&&($available == 1)&&( $used / $total < 0.98)) { 148 $acceptable_volume{$name} = 1; 149 } 150 else { 151 print "## $name $allocate $available $used $total\n"; 152 $acceptable_volume{$name} = 0; 153 } 154 if ($name =~ /ippb05/) { 155 $acceptable_volume{$name} = 0; 156 } 157 # if (($name eq 'ipp106.0')||($name eq 'ipp106.1')) { 158 # $acceptable_volume{$name} = 0; 159 # } 160 } 161 162 ## Finally, generate lists containing which volumes are located at which site. 163 ## This allows us to randomly select a volume from the list for the site that 164 ## we plan on replicating to. 165 my %volume_lists = (); 166 foreach my $vol_key (keys %requirement_map) { 167 @{ $volume_lists{$vol_key} } = grep { $acceptable_volume{$_} == 1 } ( 168 grep { $volume_map{$_} eq $vol_key } (keys %volume_map) 169 ); 170 print "$vol_key " . join(' ', @{ $volume_lists{$vol_key} }) . "\n"; 171 172 if ($#{ $volume_lists{$vol_key} } == -1) { 173 die "No acceptable volume found for site $vol_key!\n"; 174 } 175 } 176 90 177 # Pull data from the gpc1 database 91 178 my $verbose = 0; 92 179 my $mdcParser = PS::IPP::Metadata::Config->new; 93 180 94 my $files; 95 if ($stage eq 'camera') { 181 # Not technically imfiles, but the nebulous check block already uses files as a variable. 182 my $imfiles; 183 if (($stage eq 'camera')||($stage eq 'cam')) { 184 $stage = 'camera'; 96 185 my $cmd = "$camtool -processedexp -cam_id $stage_id -dbname $dbname"; 97 186 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 101 190 &my_die("Unable to perform stagetool: $error_code", $stage_id); 102 191 } 103 $ files = $mdcParser->parse_list(join "", @$stdout_buf) or192 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 104 193 &my_die("Unable to parse metadata from stagetool", $stage_id); 105 194 } … … 112 201 &my_die("Unable to perform stagetool: $error_code", $stage_id); 113 202 } 114 $ files = $mdcParser->parse_list(join "", @$stdout_buf) or203 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 115 204 &my_die("Unable to parse metadata from stagetool", $stage_id); 116 205 } … … 123 212 &my_die("Unable to perform stagetool: $error_code", $stage_id); 124 213 } 125 $ files = $mdcParser->parse_list(join "", @$stdout_buf) or214 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 126 215 &my_die("Unable to parse metadata from stagetool", $stage_id); 127 216 } … … 134 223 &my_die("Unable to perform stagetool: $error_code", $stage_id); 135 224 } 136 $ files = $mdcParser->parse_list(join "", @$stdout_buf) or225 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 137 226 &my_die("Unable to parse metadata from stagetool", $stage_id); 138 227 } 139 140 my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'], 228 elsif ($stage eq 'ff') { 229 230 my $cmd = "$fftool -summary -ff_id $stage_id -dbname $dbname"; 231 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 232 run(command => $cmd, verbose => 0); 233 unless ($success) { 234 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 235 &my_die("Unable to perform stagetool: $error_code", $stage_id); 236 } 237 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 238 &my_die("Unable to parse metadata from stagetool", $stage_id); 239 } 240 elsif ($stage eq 'diff') { 241 my $cmd = "$difftool -diffskyfile -diff_id $stage_id -dbname $dbname"; 242 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 243 run(command => $cmd, verbose => 0); 244 unless ($success) { 245 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 246 &my_die("Unable to perform stagetool: $error_code", $stage_id); 247 } 248 $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or 249 &my_die("Unable to parse metadata from stagetool", $stage_id); 250 } 251 252 my $timer_start = time(); 253 my $timer = time(); 254 my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK','PSPHOT.BACKMDL','PSPHOT.PSF.RAW.SAVE'], 141 255 'warp' => ['PSWARP.OUTPUT','PSWARP.OUTPUT.MASK','PSWARP.OUTPUT.VARIANCE'], 142 256 'stack' => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP', 143 257 'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'], 144 'skycal' => ['PSASTRO.OUTPUT.CMF']); 145 146 147 foreach my $entry (@$files) { 258 'skycal' => ['PSASTRO.OUTPUT.CMF','PSPHOT.OUTPUT.CFF'], 259 'ff' => ['PSPHOT.OUT.CMF.MEF','PSPHOT.OUTPUT.CFF','PSPHOT.FULLFORCE.OUTPUT'], 260 'diff' => ['PPSUB.OUTPUT.SOURCES','PPSUB.INVERSE.SOURCES','PSPHOT.PSF.SKY.SAVE','PSPHOT.BACKMDL.MEF','PPSUB.OUTPUT.KERNELS'] 261 ); 262 263 print "## permcheck.pl: $stage $stage_id $dbname $do_cull $do_ops $timer_start\n"; 264 foreach my $entry (@$imfiles) { 148 265 my $path_base = $entry->{path_base}; 149 266 my $data_state = $entry->{state}; 150 267 my $hostname = $entry->{hostname}; 151 268 my $quality = $entry->{quality}; 269 270 $timer = time() - $timer_start; 271 print "# $path_base $data_state $hostname $quality T: $timer\n"; 152 272 if ($quality != 0) { next; } 153 print "# $path_base $data_state $hostname $quality\n"; 273 274 my @keys = (); 154 275 foreach my $product (@{ $components{$stage} }) { 155 my @keys = (); 156 if (($stage eq 'camera')&&($product eq 'PSASTRO.OUTPUT.MASK')) { 157 my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06', 158 'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17', 159 'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27', 160 'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37', 161 'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47', 162 'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57', 163 'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67', 164 'XY71','XY72','XY73','XY74','XY75','XY76'); 165 foreach my $ota (@otas) { 166 push @keys, $ipprc->filename($product,$path_base,$ota); 276 print "# $product\n"; 277 my $is_done = 0; 278 if ($stage eq 'camera') { 279 if ($product eq 'PSASTRO.OUTPUT.MASK') { 280 my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06', 281 'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17', 282 'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27', 283 'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37', 284 'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47', 285 'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57', 286 'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67', 287 'XY71','XY72','XY73','XY74','XY75','XY76'); 288 foreach my $ota (@otas) { 289 push @keys, $ipprc->filename($product,$path_base,$ota); 290 } 291 $is_done = 1; 292 } 293 elsif (($product eq 'PSPHOT.BACKMDL')||($product eq 'PSPHOT.PSF.RAW.SAVE')) { 294 my $chip_id = $entry->{chip_id}; 295 my $chip_cmd = "$chiptool -processedimfile -chip_id $chip_id -dbname $dbname"; 296 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 297 run(command => $chip_cmd, verbose => 0); 298 unless ($success) { 299 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 300 &my_die("Unable to perform stagetool: $error_code", $stage_id); 301 } 302 my $chip_files = $mdcParser->parse_list(join "", @$stdout_buf) or 303 &my_die("Unable to parse metadata from stagetool", $stage_id); 304 305 foreach my $chip_entry (@$chip_files) { 306 my $chip_path_base = $chip_entry->{path_base}; 307 my $class_id = $chip_entry->{class_id}; 308 push @keys, $ipprc->filename($product,$chip_path_base,$class_id); 309 } 310 $is_done = 1; 167 311 } 168 312 } 169 else { 313 elsif ($stage eq 'ff') { 314 if ($product ne 'PSPHOT.FULLFORCE.OUTPUT') { 315 my $ffsum_cmd = "$fftool -result -ff_id $stage_id -dbname $dbname"; 316 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 317 run(command => $ffsum_cmd, verbose => 0); 318 unless ($success) { 319 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 320 &my_die("Unable to perform stagetool: $error_code", $stage_id); 321 } 322 my $ff_files = $mdcParser->parse_list(join "", @$stdout_buf) or 323 &my_die("Unable to parse metadata from stagetool", $stage_id); 324 325 foreach my $ff_entry (@$ff_files) { 326 my $ff_path_base = $ff_entry->{path_base}; 327 push @keys, $ipprc->filename($product,$ff_path_base); 328 } 329 $is_done = 1; 330 } 331 332 } 333 if ($is_done == 0) { 170 334 push @keys, $ipprc->filename($product,$path_base); 335 # print "$product\n"; 336 # print ">> " . $ipprc->filename($product,$path_base) . "\n"; 171 337 } 172 338 # Do validation 339 } 340 341 $timer = time() - $timer_start; 342 343 printf("# Identified %d : %s\n",$#keys + 1, $timer); 344 foreach my $key (@keys) { 345 # Testing code. 346 # print "$key\n"; 347 # next; 348 # } 349 350 # neb-stat level handling 351 my $stat = $neb->stat($key); 352 die "nebulous key: $key not found" unless $stat; 353 my $instances; 354 my $md5sum = ''; 355 356 # # This needs to be in an eval, because although we expect things to exist, they 357 # # may not. This is a fatal error in rawcheck, but need not be here. 358 eval { 359 $instances = $neb->find_instances($key, 'any'); 360 }; 361 unless (defined($instances)) { print "## skipping due to zero instances\n"; next; } 362 die "no instances found" unless $instances; 363 364 my $existing_copies = 0; 365 my $Ngood = 0; 366 my $Nbad = 0; 367 my %good_instances = (); 368 my %bad_instances = (); 369 370 #unsafe 371 my $continue_checking_md5sums = 1; 372 my %good_instances_unchecked = (); 373 #end unsafe 374 375 my @files = map {URI->new($_)->file if $_} @$instances; 376 my @validation = (); 377 378 for (my $i = 0; $i <= $#files; $i++) { 379 my ($instance_exists,$instance_md5sum,$instance_host,$instance_volume,$instance_site, $is_good); 380 ($instance_host,$instance_volume) = parse_volume($files[$i]); 381 $instance_site = $volume_map{$instance_volume}; 382 $is_good = 0; 383 384 if (-e $files[$i]) { 385 $instance_exists = 1; 386 $existing_copies++; 387 if ($continue_checking_md5sums == 1) { #unsafe 388 $instance_md5sum = local_md5sum($files[$i]); 389 390 # This is bad, but I don't know what the right solution is. We don't have 391 # the md5sum a priori. I think this is also the only major change needed 392 # from rawcheck.pl 393 if (($md5sum eq '')&&($instance_md5sum ne 'd41d8cd98f00b204e9800998ecf8427e')) { 394 $md5sum = $instance_md5sum; 395 } 396 397 if ($instance_md5sum eq $md5sum) { 398 push @{ $good_instances{$instance_site} }, $i; 399 $is_good = 1; 400 $Ngood++; 401 } 402 else { 403 push @{ $bad_instances{$instance_site} }, $i; 404 $Nbad++; 405 } 406 } #unsafe 407 else { #unsafe 408 $instance_md5sum = 'CHECKING_STOPPED'; 409 push @{ $good_instances_unchecked{$instance_site} }, $i; 410 $is_good = 1; 411 $Ngood++; 412 } #end unsafe 413 } 414 else { 415 $instance_exists = 0; 416 $instance_md5sum = 'NON-EXISTANT'; 417 push @{ $bad_instances{$instance_site} }, $i; 418 } 419 420 $validation[$i] = sprintf(" % 3d %d %32s %s %s %s\n", 421 $instance_exists,$is_good,$instance_md5sum, 422 $files[$i],$instance_host,$instance_volume); 423 424 #unsafe 425 # Remove this section when we decide to go back to actually checking everything 426 # to make sure we don't have any secret data corruption 427 if ($Ngood > 0) { 428 print "# Breaking from validation check, as a valid copy has been found.\n"; 429 $continue_checking_md5sums = 0; 430 } 431 #end unsafe 432 } 433 434 $timer = time() - $timer_start; 435 # object_id ext_id epoch available existing total timer 436 printf("%s %s %s %d %d %d %d\n",@$stat[0],@$stat[1],@$stat[4],@$stat[6],$existing_copies,@$stat[7],$timer); 437 # instance_exists is_good instance_md5sum file instance_host instance_volume 438 my $val_string = join('',@validation); 439 print "$val_string"; 440 441 # # Decide what to do 173 442 174 foreach my $key (@keys) { 175 # neb-stat level handling 176 my $stat = $neb->stat($key); 177 die "nebulous key: $key not found" unless $stat; 178 my $instances; 179 eval { 180 $instances = $neb->find_instances($key, 'any'); 181 }; 182 unless (defined($instances)) { print "## skipping due to zero instances\n"; next; } 183 die "no instances found" unless $instances; 184 185 my $user_copies; 186 eval { 187 $user_copies = $neb->getxattr($key, "user.copies"); 188 }; 189 unless(defined($user_copies)) { 190 $user_copies = 1; 191 } 192 193 my $md5sum; 194 my @validation; 195 my %md5sum_uniq; 196 my $existing_copies = 0; 197 198 my @existance; 199 my @md5sums; 200 my @diskfiles = map {URI->new($_)->file if $_} @$instances; 201 my @diskvols; 202 my @diskhosts; 203 my @quality; 204 my $Ngood = 0; 205 my $quality_mask = 0; 206 207 for (my $i = 0; $i <= $#diskfiles; $i++) { 208 if (-e $diskfiles[$i]) { 209 $existance[$i] = 1; 210 $existing_copies++; 211 $md5sums[$i] = local_md5sum($diskfiles[$i]); 212 $md5sum_uniq{$md5sums[$i]} = 1; 443 if ($Ngood == 0) { 444 # If we have no copies, we can't do anything. 445 die "No valid instance of key: $key\n"; 446 } 447 ## We have more than zero bad copies. We may cull some of these in the future, but we should try to 448 ## leave everything in the best state possible. 449 printf(">> %d %d\n",$Ngood, $#files + 1); 450 if ($Nbad > 0) { 451 my $good_copy; 452 my $good_copy_index; 453 foreach my $site_key (keys %good_instances) { 454 if ($#{ $good_instances{$site_key} } != -1) { 455 $good_copy_index = $good_instances{$site_key}[0]; 456 $good_copy = $files[$good_copy_index]; 457 last; 213 458 } 214 else { 215 $existance[$i] = 0; 216 $md5sums[$i] = 'NON-EXISTANT'; 217 $md5sum_uniq{$md5sums[$i]} = 1; 218 } 219 ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]); 220 $validation[$i] = sprintf("% 3d %32s %s %s %d", 221 $existance[$i], 222 $md5sums[$i], 223 $diskfiles[$i], 224 $diskhosts[$i],$diskvols[$i] 225 ); 226 227 # Pre-parse decisions 228 if ($existance[$i] == 0) { 229 $quality[$i] = 0; 230 } 231 elsif (is_backup_volume($diskhosts[$i])) { 232 $quality[$i] = 1; 233 $quality_mask = $quality_mask | 1; 234 $Ngood++; 235 } 236 else { 237 $quality[$i] = 2; 238 $Ngood++; 239 } 240 } 241 if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth. 242 die "There are multiple md5sum values for $key"; 243 } 244 $md5sum = (keys(%md5sum_uniq))[0]; 245 246 print "\n$key $data_state $md5sum $hostname\n"; 247 if (1) { 248 249 print 250 "object id: ", @$stat[0], "\n", 251 "key: ", @$stat[1], "\n"; 252 print 253 "epoch: ", @$stat[4], "\n"; 254 print 255 "md5sum count: ", scalar(keys %md5sum_uniq), "\n"; 256 print 257 "requested instances: ", $user_copies, "\n", 258 "available instances: ", @$stat[6], "\n", 259 "existing instances: ", $existing_copies, "\n", 260 "total instances: ", @$stat[7], "\n", 261 "instance location:\n", " " x 4; 262 print 263 join("\n" . " " x 4, @validation), "\n"; 264 } 265 266 # Decide what to do 267 if ($Ngood == 0) { 268 # DO something to attempt to fix this. 269 my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; 270 271 open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); 272 my $good_file = ''; 273 while (<DD>) { 274 $_ =~ s/^\s+//; 275 my ($z,undef,$ff) = split /\s+/; 276 if (($ff)&&(-e $ff)) { 277 my $md_response = `md5sum $ff`; 278 if ($md_response =~ /$md5sum/) { 279 $good_file = (split /\s+/,$md_response)[1]; 280 } 459 } 460 printf(">> GOOD: $good_copy\n"); 461 foreach my $site_key (keys %bad_instances) { 462 foreach my $bad_copy_index (@{ $bad_instances{$site_key} }) { 463 print "cp $good_copy $files[$bad_copy_index]\n"; 464 if ($do_ops) { 465 system("cp $good_copy $files[$bad_copy_index]"); 466 } 467 my $tmpmd5 = local_md5sum($files[$bad_copy_index]); 468 if ($tmpmd5 ne $md5sum) { 469 ## This isn't super critical, so we don't need to die here. 470 warn "Post-repair md5sum does not match! $tmpmd5 != $md5sum: $files[$bad_copy_index]"; 471 } 472 else { 473 ## success 474 push @{ $good_instances{$site_key} }, $bad_copy_index; 281 475 } 282 476 } 283 close(DD); 284 if ($good_file eq '') { 285 die "No valid instance of key: $key"; 286 } 287 else { 288 $quality[0] = 1; 289 print "cp $good_file $diskfiles[0]\n"; 290 vsystem("cp $good_file $diskfiles[0]"); 291 } 292 # Begin my best validation thought 293 { 294 my $tmpmd5 = local_md5sum($diskfiles[0]); 477 } 478 ## We've done work here, so we can't do a cull this iteration. 479 if ($do_cull == 1) { $do_cull = -1; } 480 } 481 482 #unsafe 483 foreach my $site_key (keys %good_instances_unchecked) { 484 push @{ $good_instances{$site_key} }, @{ $good_instances_unchecked{$site_key} }; 485 } 486 #end unsafe 487 488 ## We can now attempt to make replicated copies to the sites that require additional copies. 489 foreach my $site_key (keys %requirement_map) { 490 my $have_instances = $#{ $good_instances{$site_key} } + 1; 491 print "## $site_key $have_instances $requirement_map{$site_key}\n"; 492 if ($#{ $good_instances{$site_key} } + 1 < $requirement_map{$site_key}) { 493 my $rep_vol = get_random_site_volume($site_key); 494 print "neb-replicate --volume $rep_vol $key\n"; 495 if ($do_ops) { 496 $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; 497 if ($@) { die $@; } 498 499 # Begin my best validation thought 500 system("sync") == 0 or die "Couldn't sync?"; 501 my $uris = $neb->find_instances($key,$rep_vol); 502 @$uris = map {URI->new($_)->file if $_} @$uris; 503 my $tmpmd5 = local_md5sum(${ $uris }[0]); 504 505 my $validation_str = sprintf("% 3d %d %32s %s %s %s", 506 -1,-1,$tmpmd5, 507 ${ $uris }[0],"repl",$rep_vol); 508 print 509 join("\n" . " " x 4, $validation_str), "\n"; 510 295 511 if ($tmpmd5 ne $md5sum) { 296 512 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 297 513 } 298 } 299 # End my best validation thought. 300 301 $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. 302 } 303 304 if ($quality[0] == 0) { # The first instance is bad. 305 # But since we're here, and not up there, there must be at least one good copy. Find it. 306 for (my $i = 0; $i <= $#md5sums; $i++) { 307 if ($md5sums[$i] eq $md5sum) { # Found it. 308 print "cp $diskfiles[$i] $diskfiles[0]\n"; 309 vsystem("cp $diskfiles[$i] $diskfiles[0]"); 310 # Begin my best validation thought 311 { 312 my $tmpmd5 = local_md5sum($diskfiles[0]); 313 if ($tmpmd5 ne $md5sum) { 314 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 315 } 514 # End my best validation thought. 515 } 516 ## We've done work here, so we can't do a cull this iteration. 517 if ($do_cull == 1) { $do_cull = -1; } 518 } 519 } 520 521 ## Do culls if that's what we were going to do. 522 if ($do_cull == -1) { 523 die "Cull option passed, but files were modified in the scan/repair/replicate phase. Not running cull!\n"; 524 } 525 elsif ($do_cull == 1) { 526 ## At this point, we should have no files in the bad_instances lists, because we've repaired them. 527 foreach my $site_key (keys %good_instances) { 528 if ($#{ $good_instances{$site_key} } + 1 > $requirement_map{$site_key}) { 529 for ($i = $requirement_map{$site_key}; $i <= $#{ $good_instances{$site_key} }; $i++) { 530 my $cull_index = ${ $good_instances{$site_key} }[$i]; 531 my ($instance_host,$instance_volume) = parse_volume($files[$cull_index]); 532 print "neb-cull --volume $instance_volume $key\n"; 533 if ($do_ops) { 534 # The tilde here is to force hard volumes. Don't touch it. 535 # Also: the 2 is a "minimum number of copies" restriction. Let's not be crazy here. 536 $neb->cull($key,"~${instance_volume}",2) or die "failed to cull a superfluous instance"; 537 if ($@) { die "$@"; } 316 538 } 317 # End my best validation thought. 318 last; # We're done here now. 319 } 320 } 321 } 322 323 if ($Ngood == 1) { # We have only one version 324 if ($quality_mask & 1) { # ANd it's on a backup volume 325 if ($user_copies > 1) { # And we want more than one copy. 326 print "neb-replicate $key\n"; 327 if ($do_ops) { 328 $neb->replicate($key) or die "failed to replicate the single valid copy"; 329 if ($@) { die $@; } 330 } 331 } 332 } 333 else { # And it's not, so put one there 334 my $rep_vol = get_random_backup_volume(); 335 print "neb-replicate --volume $rep_vol $key\n"; 336 if ($do_ops) { 337 $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; 338 if ($@) { die $@; } 339 340 # Begin my best validation thought 341 vsystem("sync") == 0 or die "Couldn't sync?"; 342 my $uris = $neb->find_instances($key,$rep_vol); 343 @$uris = map {URI->new($_)->file if $_} @$uris; 344 my $tmpmd5 = local_md5sum(${ $uris }[0]); 345 if ($tmpmd5 ne $md5sum) { 346 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 347 } 348 # End my best validation thought. 349 } 350 } 351 } 352 else { # N >= 2 353 if (!($quality_mask & 1)) { # no backup copy 354 my $rep_vol = get_random_backup_volume(); 355 print "neb-replicate --volume $rep_vol $key\n"; 356 if ($do_ops) { 357 $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; 358 if ($@) { die $@; } 359 360 # Begin my best validation thought 361 vsystem("sync") == 0 or die "Couldn't sync?"; 362 my $uris = $neb->find_instances($key,$rep_vol); 363 @$uris = map {URI->new($_)->file if $_} @$uris; 364 my $tmpmd5 = local_md5sum(${ $uris }[0]); 365 if ($tmpmd5 ne $md5sum) { 366 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 367 } 368 # End my best validation thought. 369 } 370 } 371 if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid. 372 for (my $i = 0; $i <= $#diskfiles; $i++) { 373 if ($quality[$i] == 2) { 374 $quality[$i] = 1; 375 last; 376 } 377 } 378 } 379 } 380 #XXX NEW DEBUG THIS: This should iterate over diskfiles that are not marked with a quality = 2, and cull them. 381 # For the N>=2, we set that for the primary, and 382 # for both cases, we do not have a diskfile entry for the newly replicated copy. 383 # Therefore, this should cull down to the correct number. 384 for (my $i = 0; $i <= $#diskfiles; $i++) { 385 # print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n"; 386 if ($quality[$i] != 1) { 387 if ($existance[$i] == 0) { # This disk file doesn't exist. 388 vsystem("touch $diskfiles[$i]"); 389 } 390 my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i]; 391 print "neb-cull --volume $cull_vol $key\n"; 392 if ($do_ops) { 393 # The tilde here is to force hard volumes. Don't touch it. 394 $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance"; 395 if ($@) { die "$@"; } 396 } 397 } 398 } 399 # } 400 } # end keys for this product 401 } # end product for this entry 402 } # end entry for this id 403 404 sub vsystem { 405 my $cmd = shift; 406 print "$cmd\n"; 407 if ($do_ops) { 408 system($cmd); 409 } 410 } 539 } # End loop over extra instances 540 } # End check for sites with extra instances 541 } # End loop over sites. 542 } # End cull 543 544 } ## End loop over keys. 545 } ## Endd loop over stage components. 546 411 547 412 548 sub local_md5sum { … … 416 552 $host =~ s/\.\d//; 417 553 # print "$filename $host $volume\n"; 418 my $response = `ssh $host md5sum$filename`;554 my $response = `ssh $host remote_md5sum.pl $filename`; 419 555 chomp($response); 420 556 my ($sum, undef) = split /\s+/, $response; … … 425 561 } 426 562 427 428 429 563 sub parse_volume { 430 564 my $filename = shift(@_); 431 565 my $full_volume = (split /\//, $filename)[2]; 432 my ($hostname,$vol_index) = split /\./, $full_volume; # /; 433 return($hostname,$vol_index); 434 } 435 436 sub is_backup_volume { 437 my $hostname = shift(@_); 438 if (exists($backup_hosts{$hostname})) { 439 return(1); 440 } 441 return(0); 442 } 443 444 sub get_random_backup_volume { 445 my $NN = scalar keys %backup_destinations; 446 my $backup_host = (keys %backup_destinations)[int(rand($NN))]; 447 my $backup_vol = int(rand($backup_Nvols)); 448 449 return("${backup_host}.${backup_vol}"); 566 my ($hostname,undef) = split /\./, $full_volume; # /; 567 return($hostname,$full_volume); 568 } 569 570 sub get_random_site_volume { 571 my $site_key = shift(@_); 572 my $NN = scalar @{ $volume_lists{$site_key} }; 573 my $backup_volume = ${ $volume_lists{$site_key} }[int(rand($NN))]; 574 return($backup_volume); 450 575 } 451 576
Note:
See TracChangeset
for help on using the changeset viewer.
