Changeset 37139
- Timestamp:
- Jul 29, 2014, 12:29:08 PM (12 years ago)
- File:
-
- 1 edited
-
trunk/ippScripts/scripts/permcheck.pl (modified) (3 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/permcheck.pl
r36702 r37139 51 51 my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1, 52 52 'ippb02' => 1, 'ippb03' => 1, 53 'ippb04' => 1, 'ippb05' => 1 53 'ippb04' => 1, 'ippb05' => 1, 54 'ippb06' => 1 54 55 ); 55 my %backup_destinations = ('ippb04' => 1, 56 'ippb05' => 1); 56 my %backup_destinations = (#'ippb04' => 1, 57 #'ippb05' => 1, 58 'ippb06' => 1 59 ); 57 60 my $backup_Nvols = 3; 58 61 … … 111 114 } 112 115 113 my %components = ('camera' => ['PSASTRO.OUTPUT' ],116 my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'], 114 117 'stack' => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP', 115 118 'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'], … … 122 125 my $hostname = $entry->{hostname}; 123 126 foreach my $product (@{ $components{$stage} }) { 124 my $key = $ipprc->filename($product,$path_base); 125 127 my @keys = (); 128 if (($stage eq 'camera')&&($product eq 'PSASTRO.OUTPUT.MASK')) { 129 my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06', 130 'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17', 131 'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27', 132 'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37', 133 'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47', 134 'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57', 135 'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67', 136 'XY71','XY72','XY73','XY74','XY75','XY76'); 137 foreach my $ota (@otas) { 138 push @keys, $ipprc->filename($product,$path_base,$ota); 139 } 140 } 141 else { 142 push @keys, $ipprc->filename($product,$path_base); 143 } 126 144 # Do validation 127 128 # neb-stat level handling129 my $stat = $neb->stat($key);130 die "nebulous key: $key not found" unless $stat;131 my $instances = $neb->find_instances($key, 'any');132 die "no instances found" unless $instances;133 145 134 my $user_copies; 135 eval { 136 $user_copies = $neb->getxattr($key, "user.copies"); 137 }; 138 unless(defined($user_copies)) { 139 $user_copies = 1; 140 } 141 142 my $md5sum; 143 my @validation; 144 my %md5sum_uniq; 145 my $existing_copies = 0; 146 147 my @existance; 148 my @md5sums; 149 my @diskfiles = map {URI->new($_)->file if $_} @$instances; 150 my @diskvols; 151 my @diskhosts; 152 my @quality; 153 my $Ngood = 0; 154 my $quality_mask = 0; 155 156 for (my $i = 0; $i <= $#diskfiles; $i++) { 157 if (-e $diskfiles[$i]) { 158 $existance[$i] = 1; 159 $existing_copies++; 160 $md5sums[$i] = local_md5sum($diskfiles[$i]); 161 $md5sum_uniq{$md5sums[$i]} = 1; 162 } 163 else { 164 $existance[$i] = 0; 165 $md5sums[$i] = 'NON-EXISTANT'; 166 $md5sum_uniq{$md5sums[$i]} = 1; 167 } 168 ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]); 169 $validation[$i] = sprintf("% 3d %32s %s %s %d", 170 $existance[$i], 171 $md5sums[$i], 172 $diskfiles[$i], 173 $diskhosts[$i],$diskvols[$i] 174 ); 175 176 # Pre-parse decisions 177 if ($existance[$i] == 0) { 178 $quality[$i] = 0; 179 } 180 elsif (is_backup_volume($diskhosts[$i])) { 181 $quality[$i] = 1; 182 $quality_mask = $quality_mask | 1; 183 $Ngood++; 184 } 185 else { 186 $quality[$i] = 2; 187 $Ngood++; 188 } 189 } 190 if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth. 191 die "There are multiple md5sum values for $key"; 192 } 193 $md5sum = (keys(%md5sum_uniq))[0]; 194 195 print "\n$key $data_state $md5sum $hostname\n"; 196 if (1) { 197 198 print 199 "object id: ", @$stat[0], "\n", 200 "key: ", @$stat[1], "\n"; 201 print 202 "epoch: ", @$stat[4], "\n"; 203 print 204 "md5sum count: ", scalar(keys %md5sum_uniq), "\n"; 205 print 206 "requested instances: ", $user_copies, "\n", 207 "available instances: ", @$stat[6], "\n", 208 "existing instances: ", $existing_copies, "\n", 209 "total instances: ", @$stat[7], "\n", 210 "instance location:\n", " " x 4; 211 print 212 join("\n" . " " x 4, @validation), "\n"; 213 } 214 215 # Decide what to do 216 if ($Ngood == 0) { 217 # DO something to attempt to fix this. 218 my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; 219 220 open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); 221 my $good_file = ''; 222 while (<DD>) { 223 $_ =~ s/^\s+//; 224 my ($z,undef,$ff) = split /\s+/; 225 if (($ff)&&(-e $ff)) { 226 my $md_response = `md5sum $ff`; 227 if ($md_response =~ /$md5sum/) { 228 $good_file = (split /\s+/,$md_response)[1]; 229 } 230 } 231 } 232 close(DD); 233 if ($good_file eq '') { 234 die "No valid instance of key: $key"; 235 } 236 else { 237 $quality[0] = 1; 238 print "cp $good_file $diskfiles[0]\n"; 239 system("cp $good_file $diskfiles[0]"); 240 } 241 # Begin my best validation thought 242 { 243 my $tmpmd5 = local_md5sum($diskfiles[0]); 244 if ($tmpmd5 ne $md5sum) { 245 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 246 } 247 } 248 # End my best validation thought. 249 250 $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. 251 } 252 253 if ($quality[0] == 0) { # The first instance is bad. 254 # But since we're here, and not up there, there must be at least one good copy. Find it. 255 for (my $i = 0; $i <= $#md5sums; $i++) { 256 if ($md5sums[$i] eq $md5sum) { # Found it. 257 print "cp $diskfiles[$i] $diskfiles[0]\n"; 258 system("cp $diskfiles[$i] $diskfiles[0]"); 259 # Begin my best validation thought 260 { 261 my $tmpmd5 = local_md5sum($diskfiles[0]); 146 foreach my $key (@keys) { 147 # neb-stat level handling 148 my $stat = $neb->stat($key); 149 die "nebulous key: $key not found" unless $stat; 150 my $instances = $neb->find_instances($key, 'any'); 151 die "no instances found" unless $instances; 152 153 my $user_copies; 154 eval { 155 $user_copies = $neb->getxattr($key, "user.copies"); 156 }; 157 unless(defined($user_copies)) { 158 $user_copies = 1; 159 } 160 161 my $md5sum; 162 my @validation; 163 my %md5sum_uniq; 164 my $existing_copies = 0; 165 166 my @existance; 167 my @md5sums; 168 my @diskfiles = map {URI->new($_)->file if $_} @$instances; 169 my @diskvols; 170 my @diskhosts; 171 my @quality; 172 my $Ngood = 0; 173 my $quality_mask = 0; 174 175 for (my $i = 0; $i <= $#diskfiles; $i++) { 176 if (-e $diskfiles[$i]) { 177 $existance[$i] = 1; 178 $existing_copies++; 179 $md5sums[$i] = local_md5sum($diskfiles[$i]); 180 $md5sum_uniq{$md5sums[$i]} = 1; 181 } 182 else { 183 $existance[$i] = 0; 184 $md5sums[$i] = 'NON-EXISTANT'; 185 $md5sum_uniq{$md5sums[$i]} = 1; 186 } 187 ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]); 188 $validation[$i] = sprintf("% 3d %32s %s %s %d", 189 $existance[$i], 190 $md5sums[$i], 191 $diskfiles[$i], 192 $diskhosts[$i],$diskvols[$i] 193 ); 194 195 # Pre-parse decisions 196 if ($existance[$i] == 0) { 197 $quality[$i] = 0; 198 } 199 elsif (is_backup_volume($diskhosts[$i])) { 200 $quality[$i] = 1; 201 $quality_mask = $quality_mask | 1; 202 $Ngood++; 203 } 204 else { 205 $quality[$i] = 2; 206 $Ngood++; 207 } 208 } 209 if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth. 210 die "There are multiple md5sum values for $key"; 211 } 212 $md5sum = (keys(%md5sum_uniq))[0]; 213 214 print "\n$key $data_state $md5sum $hostname\n"; 215 if (1) { 216 217 print 218 "object id: ", @$stat[0], "\n", 219 "key: ", @$stat[1], "\n"; 220 print 221 "epoch: ", @$stat[4], "\n"; 222 print 223 "md5sum count: ", scalar(keys %md5sum_uniq), "\n"; 224 print 225 "requested instances: ", $user_copies, "\n", 226 "available instances: ", @$stat[6], "\n", 227 "existing instances: ", $existing_copies, "\n", 228 "total instances: ", @$stat[7], "\n", 229 "instance location:\n", " " x 4; 230 print 231 join("\n" . " " x 4, @validation), "\n"; 232 } 233 234 # Decide what to do 235 if ($Ngood == 0) { 236 # DO something to attempt to fix this. 237 my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; 238 239 open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); 240 my $good_file = ''; 241 while (<DD>) { 242 $_ =~ s/^\s+//; 243 my ($z,undef,$ff) = split /\s+/; 244 if (($ff)&&(-e $ff)) { 245 my $md_response = `md5sum $ff`; 246 if ($md_response =~ /$md5sum/) { 247 $good_file = (split /\s+/,$md_response)[1]; 248 } 249 } 250 } 251 close(DD); 252 if ($good_file eq '') { 253 die "No valid instance of key: $key"; 254 } 255 else { 256 $quality[0] = 1; 257 print "cp $good_file $diskfiles[0]\n"; 258 system("cp $good_file $diskfiles[0]"); 259 } 260 # Begin my best validation thought 261 { 262 my $tmpmd5 = local_md5sum($diskfiles[0]); 263 if ($tmpmd5 ne $md5sum) { 264 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 265 } 266 } 267 # End my best validation thought. 268 269 $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. 270 } 271 272 if ($quality[0] == 0) { # The first instance is bad. 273 # But since we're here, and not up there, there must be at least one good copy. Find it. 274 for (my $i = 0; $i <= $#md5sums; $i++) { 275 if ($md5sums[$i] eq $md5sum) { # Found it. 276 print "cp $diskfiles[$i] $diskfiles[0]\n"; 277 system("cp $diskfiles[$i] $diskfiles[0]"); 278 # Begin my best validation thought 279 { 280 my $tmpmd5 = local_md5sum($diskfiles[0]); 281 if ($tmpmd5 ne $md5sum) { 282 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 283 } 284 } 285 # End my best validation thought. 286 last; # We're done here now. 287 } 288 } 289 } 290 291 if ($Ngood == 1) { # We have only one version 292 if ($quality_mask & 1) { # ANd it's on a backup volume 293 print "neb-replicate $key\n"; 294 if ($do_ops) { 295 $neb->replicate($key) or die "failed to replicate the single valid copy"; 296 if ($@) { die $@; } 297 } 298 } 299 else { # And it's not, so put one there 300 my $rep_vol = get_random_backup_volume(); 301 print "neb-replicate --volume $rep_vol $key\n"; 302 if ($do_ops) { 303 $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; 304 if ($@) { die $@; } 305 306 # Begin my best validation thought 307 system("sync") == 0 or die "Couldn't sync?"; 308 my $uris = $neb->find_instances($key,$rep_vol); 309 @$uris = map {URI->new($_)->file if $_} @$uris; 310 my $tmpmd5 = local_md5sum(${ $uris }[0]); 262 311 if ($tmpmd5 ne $md5sum) { 263 312 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 264 313 } 265 } 266 # End my best validation thought. 267 last; # We're done here now. 268 } 269 } 270 } 271 272 if ($Ngood == 1) { # We have only one version 273 if ($quality_mask & 1) { # ANd it's on a backup volume 274 print "neb-replicate $key\n"; 275 if ($do_ops) { 276 $neb->replicate($key) or die "failed to replicate the single valid copy"; 277 if ($@) { die $@; } 278 } 279 } 280 else { # And it's not, so put one there 281 my $rep_vol = get_random_backup_volume(); 282 print "neb-replicate --volume $rep_vol $key\n"; 283 if ($do_ops) { 284 $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; 285 if ($@) { die $@; } 286 287 # Begin my best validation thought 288 system("sync") == 0 or die "Couldn't sync?"; 289 my $uris = $neb->find_instances($key,$rep_vol); 290 @$uris = map {URI->new($_)->file if $_} @$uris; 291 my $tmpmd5 = local_md5sum(${ $uris }[0]); 292 if ($tmpmd5 ne $md5sum) { 293 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 294 } 295 # End my best validation thought. 296 } 297 } 298 } 299 else { # N >= 2 300 if (!($quality_mask & 1)) { # no backup copy 301 my $rep_vol = get_random_backup_volume(); 302 print "neb-replicate --volume $rep_vol $key\n"; 303 if ($do_ops) { 304 $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; 305 if ($@) { die $@; } 306 307 # Begin my best validation thought 308 system("sync") == 0 or die "Couldn't sync?"; 309 my $uris = $neb->find_instances($key,$rep_vol); 310 @$uris = map {URI->new($_)->file if $_} @$uris; 311 my $tmpmd5 = local_md5sum(${ $uris }[0]); 312 if ($tmpmd5 ne $md5sum) { 313 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 314 } 315 # End my best validation thought. 316 } 317 } 318 if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid. 314 # End my best validation thought. 315 } 316 } 317 } 318 else { # N >= 2 319 if (!($quality_mask & 1)) { # no backup copy 320 my $rep_vol = get_random_backup_volume(); 321 print "neb-replicate --volume $rep_vol $key\n"; 322 if ($do_ops) { 323 $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; 324 if ($@) { die $@; } 325 326 # Begin my best validation thought 327 system("sync") == 0 or die "Couldn't sync?"; 328 my $uris = $neb->find_instances($key,$rep_vol); 329 @$uris = map {URI->new($_)->file if $_} @$uris; 330 my $tmpmd5 = local_md5sum(${ $uris }[0]); 331 if ($tmpmd5 ne $md5sum) { 332 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 333 } 334 # End my best validation thought. 335 } 336 } 337 if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid. 338 for (my $i = 0; $i <= $#diskfiles; $i++) { 339 if ($quality[$i] == 2) { 340 $quality[$i] = 1; 341 last; 342 } 343 } 344 } 319 345 for (my $i = 0; $i <= $#diskfiles; $i++) { 320 if ($quality[$i] == 2) {321 $quality[$i] = 1;322 last;323 }324 }325 }326 for (my $i = 0; $i <= $#diskfiles; $i++) {327 346 # print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n"; 328 if ($quality[$i] != 1) { 329 if ($existance[$i] == 0) { # This disk file doesn't exist. 330 system("touch $diskfiles[$i]"); 331 } 332 my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i]; 333 print "neb-cull --volume $cull_vol $key\n"; 334 if ($do_ops) { 335 # The tilde here is to force hard volumes. Don't touch it. 336 $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance"; 337 if ($@) { die "$@"; } 338 } 339 } 340 } 341 } 347 if ($quality[$i] != 1) { 348 if ($existance[$i] == 0) { # This disk file doesn't exist. 349 system("touch $diskfiles[$i]"); 350 } 351 my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i]; 352 print "neb-cull --volume $cull_vol $key\n"; 353 if ($do_ops) { 354 # The tilde here is to force hard volumes. Don't touch it. 355 $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance"; 356 if ($@) { die "$@"; } 357 } 358 } 359 } 360 } 361 } # end keys for this product 342 362 } # end product for this entry 343 363 } # end entry for this id
Note:
See TracChangeset
for help on using the changeset viewer.
