Changeset 37599 for tags/ipp-pv3-20140717
- Timestamp:
- Nov 14, 2014, 2:05:47 PM (12 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
tags/ipp-pv3-20140717/ippScripts/scripts/permcheck.pl
- Property svn:mergeinfo set to
r36702 r37599 21 21 #my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); 22 22 my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); 23 my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); 23 24 my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); 24 25 my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); … … 41 42 pod2usage( -msg => "Required options: --dbname", -exitval => 2 ) 42 43 unless $dbname; 43 pod2usage( -msg => "Required options: --stage [camera| stack|skycal]", -exitval => 2 )44 pod2usage( -msg => "Required options: --stage [camera|warp|stack|skycal]", -exitval => 2 ) 44 45 unless $stage; 45 46 pod2usage( -msg => "missing key", exitval => 2 ) … … 51 52 my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1, 52 53 'ippb02' => 1, 'ippb03' => 1, 53 'ippb04' => 1, 'ippb05' => 1 54 'ippb04' => 1, 'ippb05' => 1, 55 'ippb06' => 1 54 56 ); 55 my %backup_destinations = ('ippb04' => 1, 56 'ippb05' => 1); 57 my %backup_destinations = (#'ippb04' => 1, 58 #'ippb05' => 1, 59 'ippb06' => 1 60 ); 61 62 if ($stage eq 'warp') { # These things are destined to have their only copy on the stsciXX nodes. 63 %backup_hosts = ('stsci00' => 1, 'stsci01' => 1, 'stsci02' => 1, 64 #'stsci03' => 1, 65 'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1, 66 'stsci07' => 1, 'stsci08' => 1, 'stsci09' => 1, 67 'stsci10' => 1, 'stsci11' => 1, 'stsci12' => 1, 68 'stsci13' => 1, 'stsci14' => 1, 'stsci15' => 1, 'stsci16' => 1, 69 'stsci17' => 1, 'stsci18' => 1, 'stsci19' => 1); 70 %backup_destinations = %backup_hosts; 71 } 72 57 73 my $backup_Nvols = 3; 58 74 … … 110 126 &my_die("Unable to parse metadata from stagetool", $stage_id); 111 127 } 112 113 my %components = ('camera' => ['PSASTRO.OUTPUT'], 128 elsif ($stage eq 'warp') { 129 my $cmd = "$warptool -warped -warp_id $stage_id -dbname $dbname"; 130 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 131 run(command => $cmd, verbose => 0); 132 unless ($success) { 133 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 134 &my_die("Unable to perform stagetool: $error_code", $stage_id); 135 } 136 $files = $mdcParser->parse_list(join "", @$stdout_buf) or 137 &my_die("Unable to parse metadata from stagetool", $stage_id); 138 } 139 140 my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'], 141 'warp' => ['PSWARP.OUTPUT','PSWARP.OUTPUT.MASK','PSWARP.OUTPUT.VARIANCE'], 114 142 'stack' => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP', 115 143 'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'], … … 121 149 my $data_state = $entry->{state}; 122 150 my $hostname = $entry->{hostname}; 151 my $quality = $entry->{quality}; 152 if ($quality != 0) { next; } 153 print "# $path_base $data_state $hostname $quality\n"; 123 154 foreach my $product (@{ $components{$stage} }) { 124 my $key = $ipprc->filename($product,$path_base); 125 155 my @keys = (); 156 if (($stage eq 'camera')&&($product eq 'PSASTRO.OUTPUT.MASK')) { 157 my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06', 158 'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17', 159 'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27', 160 'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37', 161 'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47', 162 'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57', 163 'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67', 164 'XY71','XY72','XY73','XY74','XY75','XY76'); 165 foreach my $ota (@otas) { 166 push @keys, $ipprc->filename($product,$path_base,$ota); 167 } 168 } 169 else { 170 push @keys, $ipprc->filename($product,$path_base); 171 } 126 172 # Do validation 127 128 # neb-stat level handling129 my $stat = $neb->stat($key);130 die "nebulous key: $key not found" unless $stat;131 my $instances = $neb->find_instances($key, 'any');132 die "no instances found" unless $instances;133 173 134 my $user_copies; 135 eval { 136 $user_copies = $neb->getxattr($key, "user.copies"); 137 }; 138 unless(defined($user_copies)) { 139 $user_copies = 1; 140 } 141 142 my $md5sum; 143 my @validation; 144 my %md5sum_uniq; 145 my $existing_copies = 0; 146 147 my @existance; 148 my @md5sums; 149 my @diskfiles = map {URI->new($_)->file if $_} @$instances; 150 my @diskvols; 151 my @diskhosts; 152 my @quality; 153 my $Ngood = 0; 154 my $quality_mask = 0; 155 156 for (my $i = 0; $i <= $#diskfiles; $i++) { 157 if (-e $diskfiles[$i]) { 158 $existance[$i] = 1; 159 $existing_copies++; 160 $md5sums[$i] = local_md5sum($diskfiles[$i]); 161 $md5sum_uniq{$md5sums[$i]} = 1; 162 } 163 else { 164 $existance[$i] = 0; 165 $md5sums[$i] = 'NON-EXISTANT'; 166 $md5sum_uniq{$md5sums[$i]} = 1; 167 } 168 ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]); 169 $validation[$i] = sprintf("% 3d %32s %s %s %d", 170 $existance[$i], 171 $md5sums[$i], 172 $diskfiles[$i], 173 $diskhosts[$i],$diskvols[$i] 174 ); 175 176 # Pre-parse decisions 177 if ($existance[$i] == 0) { 178 $quality[$i] = 0; 179 } 180 elsif (is_backup_volume($diskhosts[$i])) { 181 $quality[$i] = 1; 182 $quality_mask = $quality_mask | 1; 183 $Ngood++; 184 } 185 else { 186 $quality[$i] = 2; 187 $Ngood++; 188 } 189 } 190 if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth. 191 die "There are multiple md5sum values for $key"; 192 } 193 $md5sum = (keys(%md5sum_uniq))[0]; 194 195 print "\n$key $data_state $md5sum $hostname\n"; 196 if (1) { 197 198 print 199 "object id: ", @$stat[0], "\n", 200 "key: ", @$stat[1], "\n"; 201 print 202 "epoch: ", @$stat[4], "\n"; 203 print 204 "md5sum count: ", scalar(keys %md5sum_uniq), "\n"; 205 print 206 "requested instances: ", $user_copies, "\n", 207 "available instances: ", @$stat[6], "\n", 208 "existing instances: ", $existing_copies, "\n", 209 "total instances: ", @$stat[7], "\n", 210 "instance location:\n", " " x 4; 211 print 212 join("\n" . " " x 4, @validation), "\n"; 213 } 214 215 # Decide what to do 216 if ($Ngood == 0) { 217 # DO something to attempt to fix this. 218 my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; 219 220 open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); 221 my $good_file = ''; 222 while (<DD>) { 223 $_ =~ s/^\s+//; 224 my ($z,undef,$ff) = split /\s+/; 225 if (($ff)&&(-e $ff)) { 226 my $md_response = `md5sum $ff`; 227 if ($md_response =~ /$md5sum/) { 228 $good_file = (split /\s+/,$md_response)[1]; 229 } 230 } 231 } 232 close(DD); 233 if ($good_file eq '') { 234 die "No valid instance of key: $key"; 235 } 236 else { 237 $quality[0] = 1; 238 print "cp $good_file $diskfiles[0]\n"; 239 system("cp $good_file $diskfiles[0]"); 240 } 241 # Begin my best validation thought 242 { 243 my $tmpmd5 = local_md5sum($diskfiles[0]); 244 if ($tmpmd5 ne $md5sum) { 245 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 246 } 247 } 248 # End my best validation thought. 249 250 $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. 251 } 252 253 if ($quality[0] == 0) { # The first instance is bad. 254 # But since we're here, and not up there, there must be at least one good copy. Find it. 255 for (my $i = 0; $i <= $#md5sums; $i++) { 256 if ($md5sums[$i] eq $md5sum) { # Found it. 257 print "cp $diskfiles[$i] $diskfiles[0]\n"; 258 system("cp $diskfiles[$i] $diskfiles[0]"); 259 # Begin my best validation thought 260 { 261 my $tmpmd5 = local_md5sum($diskfiles[0]); 174 foreach my $key (@keys) { 175 # neb-stat level handling 176 my $stat = $neb->stat($key); 177 die "nebulous key: $key not found" unless $stat; 178 my $instances; 179 eval { 180 $instances = $neb->find_instances($key, 'any'); 181 }; 182 unless (defined($instances)) { print "## skipping due to zero instances\n"; next; } 183 die "no instances found" unless $instances; 184 185 my $user_copies; 186 eval { 187 $user_copies = $neb->getxattr($key, "user.copies"); 188 }; 189 unless(defined($user_copies)) { 190 $user_copies = 1; 191 } 192 193 my $md5sum; 194 my @validation; 195 my %md5sum_uniq; 196 my $existing_copies = 0; 197 198 my @existance; 199 my @md5sums; 200 my @diskfiles = map {URI->new($_)->file if $_} @$instances; 201 my @diskvols; 202 my @diskhosts; 203 my @quality; 204 my $Ngood = 0; 205 my $quality_mask = 0; 206 207 for (my $i = 0; $i <= $#diskfiles; $i++) { 208 if (-e $diskfiles[$i]) { 209 $existance[$i] = 1; 210 $existing_copies++; 211 $md5sums[$i] = local_md5sum($diskfiles[$i]); 212 $md5sum_uniq{$md5sums[$i]} = 1; 213 } 214 else { 215 $existance[$i] = 0; 216 $md5sums[$i] = 'NON-EXISTANT'; 217 $md5sum_uniq{$md5sums[$i]} = 1; 218 } 219 ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]); 220 $validation[$i] = sprintf("% 3d %32s %s %s %d", 221 $existance[$i], 222 $md5sums[$i], 223 $diskfiles[$i], 224 $diskhosts[$i],$diskvols[$i] 225 ); 226 227 # Pre-parse decisions 228 if ($existance[$i] == 0) { 229 $quality[$i] = 0; 230 } 231 elsif (is_backup_volume($diskhosts[$i])) { 232 $quality[$i] = 1; 233 $quality_mask = $quality_mask | 1; 234 $Ngood++; 235 } 236 else { 237 $quality[$i] = 2; 238 $Ngood++; 239 } 240 } 241 if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth. 242 die "There are multiple md5sum values for $key"; 243 } 244 $md5sum = (keys(%md5sum_uniq))[0]; 245 246 print "\n$key $data_state $md5sum $hostname\n"; 247 if (1) { 248 249 print 250 "object id: ", @$stat[0], "\n", 251 "key: ", @$stat[1], "\n"; 252 print 253 "epoch: ", @$stat[4], "\n"; 254 print 255 "md5sum count: ", scalar(keys %md5sum_uniq), "\n"; 256 print 257 "requested instances: ", $user_copies, "\n", 258 "available instances: ", @$stat[6], "\n", 259 "existing instances: ", $existing_copies, "\n", 260 "total instances: ", @$stat[7], "\n", 261 "instance location:\n", " " x 4; 262 print 263 join("\n" . " " x 4, @validation), "\n"; 264 } 265 266 # Decide what to do 267 if ($Ngood == 0) { 268 # DO something to attempt to fix this. 269 my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; 270 271 open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); 272 my $good_file = ''; 273 while (<DD>) { 274 $_ =~ s/^\s+//; 275 my ($z,undef,$ff) = split /\s+/; 276 if (($ff)&&(-e $ff)) { 277 my $md_response = `md5sum $ff`; 278 if ($md_response =~ /$md5sum/) { 279 $good_file = (split /\s+/,$md_response)[1]; 280 } 281 } 282 } 283 close(DD); 284 if ($good_file eq '') { 285 die "No valid instance of key: $key"; 286 } 287 else { 288 $quality[0] = 1; 289 print "cp $good_file $diskfiles[0]\n"; 290 vsystem("cp $good_file $diskfiles[0]"); 291 } 292 # Begin my best validation thought 293 { 294 my $tmpmd5 = local_md5sum($diskfiles[0]); 295 if ($tmpmd5 ne $md5sum) { 296 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 297 } 298 } 299 # End my best validation thought. 300 301 $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. 302 } 303 304 if ($quality[0] == 0) { # The first instance is bad. 305 # But since we're here, and not up there, there must be at least one good copy. Find it. 306 for (my $i = 0; $i <= $#md5sums; $i++) { 307 if ($md5sums[$i] eq $md5sum) { # Found it. 308 print "cp $diskfiles[$i] $diskfiles[0]\n"; 309 vsystem("cp $diskfiles[$i] $diskfiles[0]"); 310 # Begin my best validation thought 311 { 312 my $tmpmd5 = local_md5sum($diskfiles[0]); 313 if ($tmpmd5 ne $md5sum) { 314 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 315 } 316 } 317 # End my best validation thought. 318 last; # We're done here now. 319 } 320 } 321 } 322 323 if ($Ngood == 1) { # We have only one version 324 if ($quality_mask & 1) { # ANd it's on a backup volume 325 if ($user_copies > 1) { # And we want more than one copy. 326 print "neb-replicate $key\n"; 327 if ($do_ops) { 328 $neb->replicate($key) or die "failed to replicate the single valid copy"; 329 if ($@) { die $@; } 330 } 331 } 332 } 333 else { # And it's not, so put one there 334 my $rep_vol = get_random_backup_volume(); 335 print "neb-replicate --volume $rep_vol $key\n"; 336 if ($do_ops) { 337 $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; 338 if ($@) { die $@; } 339 340 # Begin my best validation thought 341 vsystem("sync") == 0 or die "Couldn't sync?"; 342 my $uris = $neb->find_instances($key,$rep_vol); 343 @$uris = map {URI->new($_)->file if $_} @$uris; 344 my $tmpmd5 = local_md5sum(${ $uris }[0]); 262 345 if ($tmpmd5 ne $md5sum) { 263 346 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 264 347 } 265 } 266 # End my best validation thought. 267 last; # We're done here now. 268 } 269 } 270 } 271 272 if ($Ngood == 1) { # We have only one version 273 if ($quality_mask & 1) { # ANd it's on a backup volume 274 print "neb-replicate $key\n"; 275 if ($do_ops) { 276 $neb->replicate($key) or die "failed to replicate the single valid copy"; 277 if ($@) { die $@; } 278 } 279 } 280 else { # And it's not, so put one there 281 my $rep_vol = get_random_backup_volume(); 282 print "neb-replicate --volume $rep_vol $key\n"; 283 if ($do_ops) { 284 $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; 285 if ($@) { die $@; } 286 287 # Begin my best validation thought 288 system("sync") == 0 or die "Couldn't sync?"; 289 my $uris = $neb->find_instances($key,$rep_vol); 290 @$uris = map {URI->new($_)->file if $_} @$uris; 291 my $tmpmd5 = local_md5sum(${ $uris }[0]); 292 if ($tmpmd5 ne $md5sum) { 293 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 294 } 295 # End my best validation thought. 296 } 297 } 298 } 299 else { # N >= 2 300 if (!($quality_mask & 1)) { # no backup copy 301 my $rep_vol = get_random_backup_volume(); 302 print "neb-replicate --volume $rep_vol $key\n"; 303 if ($do_ops) { 304 $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; 305 if ($@) { die $@; } 306 307 # Begin my best validation thought 308 system("sync") == 0 or die "Couldn't sync?"; 309 my $uris = $neb->find_instances($key,$rep_vol); 310 @$uris = map {URI->new($_)->file if $_} @$uris; 311 my $tmpmd5 = local_md5sum(${ $uris }[0]); 312 if ($tmpmd5 ne $md5sum) { 313 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 314 } 315 # End my best validation thought. 316 } 317 } 318 if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid. 319 for (my $i = 0; $i <= $#diskfiles; $i++) { 320 if ($quality[$i] == 2) { 321 $quality[$i] = 1; 322 last; 323 } 324 } 325 } 348 # End my best validation thought. 349 } 350 } 351 } 352 else { # N >= 2 353 if (!($quality_mask & 1)) { # no backup copy 354 my $rep_vol = get_random_backup_volume(); 355 print "neb-replicate --volume $rep_vol $key\n"; 356 if ($do_ops) { 357 $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; 358 if ($@) { die $@; } 359 360 # Begin my best validation thought 361 vsystem("sync") == 0 or die "Couldn't sync?"; 362 my $uris = $neb->find_instances($key,$rep_vol); 363 @$uris = map {URI->new($_)->file if $_} @$uris; 364 my $tmpmd5 = local_md5sum(${ $uris }[0]); 365 if ($tmpmd5 ne $md5sum) { 366 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; 367 } 368 # End my best validation thought. 369 } 370 } 371 if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid. 372 for (my $i = 0; $i <= $#diskfiles; $i++) { 373 if ($quality[$i] == 2) { 374 $quality[$i] = 1; 375 last; 376 } 377 } 378 } 379 } 380 #XXX NEW DEBUG THIS: This should iterate over diskfiles that are not marked with a quality = 2, and cull them. 381 # For the N>=2, we set that for the primary, and 382 # for both cases, we do not have a diskfile entry for the newly replicated copy. 383 # Therefore, this should cull down to the correct number. 326 384 for (my $i = 0; $i <= $#diskfiles; $i++) { 327 385 # print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n"; 328 386 if ($quality[$i] != 1) { 329 387 if ($existance[$i] == 0) { # This disk file doesn't exist. 330 system("touch $diskfiles[$i]");388 vsystem("touch $diskfiles[$i]"); 331 389 } 332 390 my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i]; … … 339 397 } 340 398 } 341 } 399 # } 400 } # end keys for this product 342 401 } # end product for this entry 343 402 } # end entry for this id 403 404 sub vsystem { 405 my $cmd = shift; 406 print "$cmd\n"; 407 if ($do_ops) { 408 system($cmd); 409 } 410 } 344 411 345 412 sub local_md5sum { … … 359 426 360 427 428 361 429 sub parse_volume { 362 430 my $filename = shift(@_);
Note:
See TracChangeset
for help on using the changeset viewer.
