Index: /tags/ipp-pv3-20140717/ippScripts/scripts/permcheck.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/permcheck.pl (revision 37598) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/permcheck.pl (revision 37599) @@ -21,4 +21,5 @@ #my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); @@ -41,5 +42,5 @@ pod2usage( -msg => "Required options: --dbname", -exitval => 2 ) unless $dbname; -pod2usage( -msg => "Required options: --stage [camera|stack|skycal]", -exitval => 2 ) +pod2usage( -msg => "Required options: --stage [camera|warp|stack|skycal]", -exitval => 2 ) unless $stage; pod2usage( -msg => "missing key", exitval => 2 ) @@ -51,8 +52,23 @@ my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1, 'ippb02' => 1, 'ippb03' => 1, - 'ippb04' => 1, 'ippb05' => 1 + 'ippb04' => 1, 'ippb05' => 1, + 'ippb06' => 1 ); -my %backup_destinations = ('ippb04' => 1, - 'ippb05' => 1); +my %backup_destinations = (#'ippb04' => 1, + #'ippb05' => 1, + 'ippb06' => 1 +); + +if ($stage eq 'warp') { # These things are destined to have their only copy on the stsciXX nodes. + %backup_hosts = ('stsci00' => 1, 'stsci01' => 1, 'stsci02' => 1, + #'stsci03' => 1, + 'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1, + 'stsci07' => 1, 'stsci08' => 1, 'stsci09' => 1, + 'stsci10' => 1, 'stsci11' => 1, 'stsci12' => 1, + 'stsci13' => 1, 'stsci14' => 1, 'stsci15' => 1, 'stsci16' => 1, + 'stsci17' => 1, 'stsci18' => 1, 'stsci19' => 1); + %backup_destinations = %backup_hosts; +} + my $backup_Nvols = 3; @@ -110,6 +126,18 @@ &my_die("Unable to parse metadata from stagetool", $stage_id); } - -my %components = ('camera' => ['PSASTRO.OUTPUT'], +elsif ($stage eq 'warp') { + my $cmd = "$warptool -warped -warp_id $stage_id -dbname $dbname"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stagetool: $error_code", $stage_id); + } + $files = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stagetool", $stage_id); +} + +my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'], + 'warp' => ['PSWARP.OUTPUT','PSWARP.OUTPUT.MASK','PSWARP.OUTPUT.VARIANCE'], 'stack' => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP', 'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'], @@ -121,212 +149,242 @@ my $data_state = $entry->{state}; my $hostname = $entry->{hostname}; + my $quality = $entry->{quality}; + if ($quality != 0) { next; } + print "# $path_base $data_state $hostname $quality\n"; foreach my $product (@{ $components{$stage} }) { - my $key = $ipprc->filename($product,$path_base); - + my @keys = (); + if (($stage eq 'camera')&&($product eq 'PSASTRO.OUTPUT.MASK')) { + my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06', + 'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17', + 'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27', + 'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37', + 'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47', + 'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57', + 'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67', + 'XY71','XY72','XY73','XY74','XY75','XY76'); + foreach my $ota (@otas) { + push @keys, $ipprc->filename($product,$path_base,$ota); + } + } + else { + push @keys, $ipprc->filename($product,$path_base); + } # Do validation - - # neb-stat level handling - my $stat = $neb->stat($key); - die "nebulous key: $key not found" unless $stat; - my $instances = $neb->find_instances($key, 'any'); - die "no instances found" unless $instances; - my $user_copies; - eval { - $user_copies = $neb->getxattr($key, "user.copies"); - }; - unless(defined($user_copies)) { - $user_copies = 1; - } - - my $md5sum; - my @validation; - my %md5sum_uniq; - my $existing_copies = 0; - - my @existance; - my @md5sums; - my @diskfiles = map {URI->new($_)->file if $_} @$instances; - my @diskvols; - my @diskhosts; - my @quality; - my $Ngood = 0; - my $quality_mask = 0; - - for (my $i = 0; $i <= $#diskfiles; $i++) { - if (-e $diskfiles[$i]) { - $existance[$i] = 1; - $existing_copies++; - $md5sums[$i] = local_md5sum($diskfiles[$i]); - $md5sum_uniq{$md5sums[$i]} = 1; - } - else { - $existance[$i] = 0; - $md5sums[$i] = 'NON-EXISTANT'; - $md5sum_uniq{$md5sums[$i]} = 1; - } - ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]); - $validation[$i] = sprintf("% 3d %32s %s %s %d", - $existance[$i], - $md5sums[$i], - $diskfiles[$i], - $diskhosts[$i],$diskvols[$i] - ); - - # Pre-parse decisions - if ($existance[$i] == 0) { - $quality[$i] = 0; - } - elsif (is_backup_volume($diskhosts[$i])) { - $quality[$i] = 1; - $quality_mask = $quality_mask | 1; - $Ngood++; - } - else { - $quality[$i] = 2; - $Ngood++; - } - } - if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth. - die "There are multiple md5sum values for $key"; - } - $md5sum = (keys(%md5sum_uniq))[0]; - - print "\n$key $data_state $md5sum $hostname\n"; - if (1) { - - print - "object id: ", @$stat[0], "\n", - "key: ", @$stat[1], "\n"; - print - "epoch: ", @$stat[4], "\n"; - print - "md5sum count: ", scalar(keys %md5sum_uniq), "\n"; - print - "requested instances: ", $user_copies, "\n", - "available instances: ", @$stat[6], "\n", - "existing instances: ", $existing_copies, "\n", - "total instances: ", @$stat[7], "\n", - "instance location:\n", " " x 4; - print - join("\n" . " " x 4, @validation), "\n"; - } - - # Decide what to do - if ($Ngood == 0) { - # DO something to attempt to fix this. - my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; - - open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); - my $good_file = ''; - while (
) { - $_ =~ s/^\s+//; - my ($z,undef,$ff) = split /\s+/; - if (($ff)&&(-e $ff)) { - my $md_response = `md5sum $ff`; - if ($md_response =~ /$md5sum/) { - $good_file = (split /\s+/,$md_response)[1]; - } - } - } - close(DD); - if ($good_file eq '') { - die "No valid instance of key: $key"; - } - else { - $quality[0] = 1; - print "cp $good_file $diskfiles[0]\n"; - system("cp $good_file $diskfiles[0]"); - } - # Begin my best validation thought - { - my $tmpmd5 = local_md5sum($diskfiles[0]); - if ($tmpmd5 ne $md5sum) { - die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; - } - } - # End my best validation thought. - - $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. - } - - if ($quality[0] == 0) { # The first instance is bad. - # But since we're here, and not up there, there must be at least one good copy. Find it. - for (my $i = 0; $i <= $#md5sums; $i++) { - if ($md5sums[$i] eq $md5sum) { # Found it. - print "cp $diskfiles[$i] $diskfiles[0]\n"; - system("cp $diskfiles[$i] $diskfiles[0]"); - # Begin my best validation thought - { - my $tmpmd5 = local_md5sum($diskfiles[0]); + foreach my $key (@keys) { + # neb-stat level handling + my $stat = $neb->stat($key); + die "nebulous key: $key not found" unless $stat; + my $instances; + eval { + $instances = $neb->find_instances($key, 'any'); + }; + unless (defined($instances)) { print "## skipping due to zero instances\n"; next; } + die "no instances found" unless $instances; + + my $user_copies; + eval { + $user_copies = $neb->getxattr($key, "user.copies"); + }; + unless(defined($user_copies)) { + $user_copies = 1; + } + + my $md5sum; + my @validation; + my %md5sum_uniq; + my $existing_copies = 0; + + my @existance; + my @md5sums; + my @diskfiles = map {URI->new($_)->file if $_} @$instances; + my @diskvols; + my @diskhosts; + my @quality; + my $Ngood = 0; + my $quality_mask = 0; + + for (my $i = 0; $i <= $#diskfiles; $i++) { + if (-e $diskfiles[$i]) { + $existance[$i] = 1; + $existing_copies++; + $md5sums[$i] = local_md5sum($diskfiles[$i]); + $md5sum_uniq{$md5sums[$i]} = 1; + } + else { + $existance[$i] = 0; + $md5sums[$i] = 'NON-EXISTANT'; + $md5sum_uniq{$md5sums[$i]} = 1; + } + ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]); + $validation[$i] = sprintf("% 3d %32s %s %s %d", + $existance[$i], + $md5sums[$i], + $diskfiles[$i], + $diskhosts[$i],$diskvols[$i] + ); + + # Pre-parse decisions + if ($existance[$i] == 0) { + $quality[$i] = 0; + } + elsif (is_backup_volume($diskhosts[$i])) { + $quality[$i] = 1; + $quality_mask = $quality_mask | 1; + $Ngood++; + } + else { + $quality[$i] = 2; + $Ngood++; + } + } + if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth. + die "There are multiple md5sum values for $key"; + } + $md5sum = (keys(%md5sum_uniq))[0]; + + print "\n$key $data_state $md5sum $hostname\n"; + if (1) { + + print + "object id: ", @$stat[0], "\n", + "key: ", @$stat[1], "\n"; + print + "epoch: ", @$stat[4], "\n"; + print + "md5sum count: ", scalar(keys %md5sum_uniq), "\n"; + print + "requested instances: ", $user_copies, "\n", + "available instances: ", @$stat[6], "\n", + "existing instances: ", $existing_copies, "\n", + "total instances: ", @$stat[7], "\n", + "instance location:\n", " " x 4; + print + join("\n" . " " x 4, @validation), "\n"; + } + + # Decide what to do + if ($Ngood == 0) { + # DO something to attempt to fix this. + my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; + + open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); + my $good_file = ''; + while (
) { + $_ =~ s/^\s+//; + my ($z,undef,$ff) = split /\s+/; + if (($ff)&&(-e $ff)) { + my $md_response = `md5sum $ff`; + if ($md_response =~ /$md5sum/) { + $good_file = (split /\s+/,$md_response)[1]; + } + } + } + close(DD); + if ($good_file eq '') { + die "No valid instance of key: $key"; + } + else { + $quality[0] = 1; + print "cp $good_file $diskfiles[0]\n"; + vsystem("cp $good_file $diskfiles[0]"); + } + # Begin my best validation thought + { + my $tmpmd5 = local_md5sum($diskfiles[0]); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + } + # End my best validation thought. + + $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. + } + + if ($quality[0] == 0) { # The first instance is bad. + # But since we're here, and not up there, there must be at least one good copy. Find it. + for (my $i = 0; $i <= $#md5sums; $i++) { + if ($md5sums[$i] eq $md5sum) { # Found it. + print "cp $diskfiles[$i] $diskfiles[0]\n"; + vsystem("cp $diskfiles[$i] $diskfiles[0]"); + # Begin my best validation thought + { + my $tmpmd5 = local_md5sum($diskfiles[0]); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + } + # End my best validation thought. + last; # We're done here now. + } + } + } + + if ($Ngood == 1) { # We have only one version + if ($quality_mask & 1) { # ANd it's on a backup volume + if ($user_copies > 1) { # And we want more than one copy. + print "neb-replicate $key\n"; + if ($do_ops) { + $neb->replicate($key) or die "failed to replicate the single valid copy"; + if ($@) { die $@; } + } + } + } + else { # And it's not, so put one there + my $rep_vol = get_random_backup_volume(); + print "neb-replicate --volume $rep_vol $key\n"; + if ($do_ops) { + $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; + if ($@) { die $@; } + + # Begin my best validation thought + vsystem("sync") == 0 or die "Couldn't sync?"; + my $uris = $neb->find_instances($key,$rep_vol); + @$uris = map {URI->new($_)->file if $_} @$uris; + my $tmpmd5 = local_md5sum(${ $uris }[0]); if ($tmpmd5 ne $md5sum) { die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; } - } - # End my best validation thought. - last; # We're done here now. - } - } - } - - if ($Ngood == 1) { # We have only one version - if ($quality_mask & 1) { # ANd it's on a backup volume - print "neb-replicate $key\n"; - if ($do_ops) { - $neb->replicate($key) or die "failed to replicate the single valid copy"; - if ($@) { die $@; } - } - } - else { # And it's not, so put one there - my $rep_vol = get_random_backup_volume(); - print "neb-replicate --volume $rep_vol $key\n"; - if ($do_ops) { - $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; - if ($@) { die $@; } - - # Begin my best validation thought - system("sync") == 0 or die "Couldn't sync?"; - my $uris = $neb->find_instances($key,$rep_vol); - @$uris = map {URI->new($_)->file if $_} @$uris; - my $tmpmd5 = local_md5sum(${ $uris }[0]); - if ($tmpmd5 ne $md5sum) { - die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; - } - # End my best validation thought. - } - } - } - else { # N >= 2 - if (!($quality_mask & 1)) { # no backup copy - my $rep_vol = get_random_backup_volume(); - print "neb-replicate --volume $rep_vol $key\n"; - if ($do_ops) { - $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; - if ($@) { die $@; } - - # Begin my best validation thought - system("sync") == 0 or die "Couldn't sync?"; - my $uris = $neb->find_instances($key,$rep_vol); - @$uris = map {URI->new($_)->file if $_} @$uris; - my $tmpmd5 = local_md5sum(${ $uris }[0]); - if ($tmpmd5 ne $md5sum) { - die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; - } - # End my best validation thought. - } - } - if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid. - for (my $i = 0; $i <= $#diskfiles; $i++) { - if ($quality[$i] == 2) { - $quality[$i] = 1; - last; - } - } - } + # End my best validation thought. + } + } + } + else { # N >= 2 + if (!($quality_mask & 1)) { # no backup copy + my $rep_vol = get_random_backup_volume(); + print "neb-replicate --volume $rep_vol $key\n"; + if ($do_ops) { + $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; + if ($@) { die $@; } + + # Begin my best validation thought + vsystem("sync") == 0 or die "Couldn't sync?"; + my $uris = $neb->find_instances($key,$rep_vol); + @$uris = map {URI->new($_)->file if $_} @$uris; + my $tmpmd5 = local_md5sum(${ $uris }[0]); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + # End my best validation thought. + } + } + if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid. + for (my $i = 0; $i <= $#diskfiles; $i++) { + if ($quality[$i] == 2) { + $quality[$i] = 1; + last; + } + } + } + } +#XXX NEW DEBUG THIS: This should iterate over diskfiles that are not marked with a quality = 2, and cull them. +# For the N>=2, we set that for the primary, and +# for both cases, we do not have a diskfile entry for the newly replicated copy. +# Therefore, this should cull down to the correct number. for (my $i = 0; $i <= $#diskfiles; $i++) { # print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n"; if ($quality[$i] != 1) { if ($existance[$i] == 0) { # This disk file doesn't exist. - system("touch $diskfiles[$i]"); + vsystem("touch $diskfiles[$i]"); } my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i]; @@ -339,7 +397,16 @@ } } - } + # } + } # end keys for this product } # end product for this entry } # end entry for this id + +sub vsystem { + my $cmd = shift; + print "$cmd\n"; + if ($do_ops) { + system($cmd); + } +} sub local_md5sum { @@ -359,4 +426,5 @@ + sub parse_volume { my $filename = shift(@_);