Index: trunk/ippScripts/scripts/permcheck.pl =================================================================== --- trunk/ippScripts/scripts/permcheck.pl (revision 39901) +++ trunk/ippScripts/scripts/permcheck.pl (revision 39955) @@ -26,4 +26,5 @@ my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1); +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); my ($server,$dbname,$stage,$stage_id,$do_cull,$save_log); @@ -55,9 +56,10 @@ unless defined $stage_id; +my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n"; + if ($save_log) { my $time = time(); - my $logDest = "neb://any/perm_check/${stage}/${stage}_${stage_id}.${time}"; - my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n"; - $ipprc->redirect_output($logDest) or die "Could not redirect output to logfile ${logDest}\n"; + my $logDest = "neb://any/perm_check/${stage}/${stage}_${stage_id}"; + $ipprc->redirect_to_logfile($logDest) or die "Could not redirect output to logfile ${logDest}\n"; } @@ -82,6 +84,6 @@ my %requirement_map = (); $requirement_map{ITC} = 1; -$requirement_map{OFFSITE} = 1; -$requirement_map{MRTCB} = 0; +$requirement_map{OFFSITE} = 0; +$requirement_map{MRTCB} = 1; ## Second, construct a list of volumes, mapped to their site location, using the @@ -113,14 +115,27 @@ $volume_map{$vol} = 'ITC'; } +for ($i = 118; $i <= 122; $i++) { + my $vol = sprintf("ipp%03d.0",$i); + $volume_map{$vol} = 'MRTCB'; + $vol = sprintf("ipp%03d.1",$i); + $volume_map{$vol} = 'MRTCB'; +} + + for ($i = 0; $i <= 15; $i++) { - if ($i == 6) { next; } # This isn't "offsite", it's with the rest of the maui cluster. + my $loc = 'OFFSITE'; + if ($i == 6) { + $loc = 'MRTCB'; + } # This isn't "offsite", it's with the rest of the maui cluster. if ($i == 9) { next; } # Not online my $vol = sprintf("ippb%02d.0",$i); - $volume_map{$vol} = 'OFFSITE'; + $volume_map{$vol} = $loc; $vol = sprintf("ippb%02d.1",$i); - $volume_map{$vol} = 'OFFSITE'; - $vol = sprintf("ippb%02d.2",$i); - $volume_map{$vol} = 'OFFSITE'; + $volume_map{$vol} = $loc; + if ($i <= 6) { + $vol = sprintf("ippb%02d.2",$i); + $volume_map{$vol} = $loc; + } } @@ -166,5 +181,6 @@ # Not technically imfiles, but the nebulous check block already uses files as a variable. my $imfiles; -if ($stage eq 'camera') { +if (($stage eq 'camera')||($stage eq 'cam')) { + $stage = 'camera'; my $cmd = "$camtool -processedexp -cam_id $stage_id -dbname $dbname"; my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -211,5 +227,6 @@ } elsif ($stage eq 'ff') { - my $cmd = "$fftool -result --ff_id $stage_id -dbname $dbname"; + + my $cmd = "$fftool -summary -ff_id $stage_id -dbname $dbname"; my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $cmd, verbose => 0); @@ -221,5 +238,15 @@ &my_die("Unable to parse metadata from stagetool", $stage_id); } - +elsif ($stage eq 'diff') { + my $cmd = "$difftool -diffskyfile -diff_id $stage_id -dbname $dbname"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stagetool: $error_code", $stage_id); + } + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stagetool", $stage_id); +} my $timer_start = time(); @@ -230,8 +257,7 @@ 'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'], 'skycal' => ['PSASTRO.OUTPUT.CMF','PSPHOT.OUTPUT.CFF'], - 'ff' => ['PSPHOT.OUT.CMF.MEF','PSPHOT.OUTPUT.CFF','PSPHOT.FULLFORCE.OUTPUT']); - -my $timer_start = time(); -my $timer = time(); + 'ff' => ['PSPHOT.OUT.CMF.MEF','PSPHOT.OUTPUT.CFF','PSPHOT.FULLFORCE.OUTPUT'], + 'diff' => ['PPSUB.OUTPUT.SOURCES','PPSUB.INVERSE.SOURCES','PSPHOT.PSF.SKY.SAVE','PSPHOT.BACKMDL.MEF','PPSUB.OUTPUT.KERNELS'] + ); print "## permcheck.pl: $stage $stage_id $dbname $do_cull $do_ops $timer_start\n"; @@ -242,5 +268,5 @@ my $quality = $entry->{quality}; - $timer = $time() - $timer_start; + $timer = time() - $timer_start; print "# $path_base $data_state $hostname $quality T: $timer\n"; if ($quality != 0) { next; } @@ -248,4 +274,6 @@ my @keys = (); foreach my $product (@{ $components{$stage} }) { + print "# $product\n"; + my $is_done = 0; if ($stage eq 'camera') { if ($product eq 'PSASTRO.OUTPUT.MASK') { @@ -261,4 +289,5 @@ push @keys, $ipprc->filename($product,$path_base,$ota); } + $is_done = 1; } elsif (($product eq 'PSPHOT.BACKMDL')||($product eq 'PSPHOT.PSF.RAW.SAVE')) { @@ -271,5 +300,5 @@ &my_die("Unable to perform stagetool: $error_code", $stage_id); } - my $chip_$files = $mdcParser->parse_list(join "", @$stdout_buf) or + my $chip_files = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata from stagetool", $stage_id); @@ -279,25 +308,31 @@ push @keys, $ipprc->filename($product,$chip_path_base,$class_id); } + $is_done = 1; + } + } + elsif ($stage eq 'ff') { + if ($product ne 'PSPHOT.FULLFORCE.OUTPUT') { + my $ffsum_cmd = "$fftool -result -ff_id $stage_id -dbname $dbname"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $ffsum_cmd, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stagetool: $error_code", $stage_id); + } + my $ff_files = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stagetool", $stage_id); + + foreach my $ff_entry (@$ff_files) { + my $ff_path_base = $ff_entry->{path_base}; + push @keys, $ipprc->filename($product,$ff_path_base); + } + $is_done = 1; } - } - elsif (($stage eq 'ff')&&($product eq 'PSPHOT.FULLFORCE.OUTPUT')) { - my $ffsum_cmd = "$fftool -summary -ff_id $stage_id -dbname $dbname"; - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $ffsum_cmd, verbose => 0); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform stagetool: $error_code", $stage_id); - } - my $chip_$files = $mdcParser->parse_list(join "", @$stdout_buf) or - &my_die("Unable to parse metadata from stagetool", $stage_id); - - foreach my $chip_entry (@$chip_files) { - my $chip_path_base = $chip_entry->{path_base}; - my $class_id = $chip_entry->{class_id}; - push @keys, $ipprc->filename($product,$chip_path_base,); - } + } - else { + if ($is_done == 0) { push @keys, $ipprc->filename($product,$path_base); +# print "$product\n"; +# print ">> " . $ipprc->filename($product,$path_base) . "\n"; } # Do validation @@ -308,204 +343,173 @@ printf("# Identified %d : %s\n",$#keys + 1, $timer); foreach my $key (@keys) { - # neb-stat level handling - my $stat = $neb->stat($key); - die "nebulous key: $key not found" unless $stat; - my $instances; - my $md5sum = ''; - - # This needs to be in an eval, because although we expect things to exist, they - # may not. This is a fatal error in rawcheck, but need not be here. - eval { - $instances = $neb->find_instances($key, 'any'); - }; - unless (defined($instances)) { print "## skipping due to zero instances\n"; next; } - die "no instances found" unless $instances; + # Testing code. + print "$key\n"; + next; + } + +# # neb-stat level handling +# my $stat = $neb->stat($key); +# die "nebulous key: $key not found" unless $stat; +# my $instances; +# my $md5sum = ''; + +# # This needs to be in an eval, because although we expect things to exist, they +# # may not. This is a fatal error in rawcheck, but need not be here. +# eval { +# $instances = $neb->find_instances($key, 'any'); +# }; +# unless (defined($instances)) { print "## skipping due to zero instances\n"; next; } +# die "no instances found" unless $instances; - my $Ngood; - my %good_instances = (); - my %bad_instances = (); - - my @files = map {URI->new($_)->file if $_} @$instances; - my @validation = (); - - for (my $i = 0; $i <= $#files; $i++) { - my ($instance_exists,$instance_md5sum,$instance_host,$instance_volume,$instance_site, $is_good); - ($instance_host,$instance_volume) = parse_volume($files[$i]); - $instance_site = $volume_map{$instance_volume}; - $is_good = 0; - - if (-e $files[$i]) { - $instance_exists = 1; - $existing_copies++; - $instance_md5sum = local_md5sum($files[$i]); - - # This is bad, but I don't know what the right solution is. We don't have - # the md5sum a priori. I think this is also the only major change needed - # from rawcheck.pl - if (($md5sum eq '')&&($instance_md5sum ne 'd41d8cd98f00b204e9800998ecf8427e')) { - $md5sum = $instance_md5sum; - } - - if ($instance_md5sum eq $md5sum) { - push @{ $good_instance{$instance_site} }, $i; - $is_good = 1; - $Ngood++; - } - else { - push @{ $bad_instances{$instance_site} }, $i; - } - } - else { - $instance_exists = 0; - $instance_md5sum = 'NON-EXISTANT'; - push @{ $bad_instances{$instance_site} }, $i; - } +# my $Ngood; +# my %good_instances = (); +# my %bad_instances = (); + +# my @files = map {URI->new($_)->file if $_} @$instances; +# my @validation = (); + +# for (my $i = 0; $i <= $#files; $i++) { +# my ($instance_exists,$instance_md5sum,$instance_host,$instance_volume,$instance_site, $is_good); +# ($instance_host,$instance_volume) = parse_volume($files[$i]); +# $instance_site = $volume_map{$instance_volume}; +# $is_good = 0; + +# if (-e $files[$i]) { +# $instance_exists = 1; +# $existing_copies++; +# $instance_md5sum = local_md5sum($files[$i]); + +# # This is bad, but I don't know what the right solution is. We don't have +# # the md5sum a priori. I think this is also the only major change needed +# # from rawcheck.pl +# if (($md5sum eq '')&&($instance_md5sum ne 'd41d8cd98f00b204e9800998ecf8427e')) { +# $md5sum = $instance_md5sum; +# } + +# if ($instance_md5sum eq $md5sum) { +# push @{ $good_instance{$instance_site} }, $i; +# $is_good = 1; +# $Ngood++; +# } +# else { +# push @{ $bad_instances{$instance_site} }, $i; +# } +# } +# else { +# $instance_exists = 0; +# $instance_md5sum = 'NON-EXISTANT'; +# push @{ $bad_instances{$instance_site} }, $i; +# } - $validation[$i] = sprintf(" % 3d %d %32s %s %s %s\n", - $instance_exists,$is_good,$instance_md5sum, - $files[$i],$instance_host,$instance_volume); - } - - $time = time() - $timer_start; - # object_id ext_id epoch available existing total timer - printf("%s %s %s %d %d %d %d\n",@$stat[0],@$stat[1],@$stat[4],@$stat[6],$existing_copies,@$stat[7],$timer); - # instance_exists is_good instance_md5sum file instance_host instance_volume - my $val_string = join('',@validation); - print "$val_string"; - - # Decide what to do - - ## This block attempts to find an out-of-nebulous instance with a good md5sum, - ## and substitutes it for the 0-th listed instance. This gives us one good - ## copy to work with. - if ($Ngood == 0) { - # DO something to attempt to fix this. - my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; - - open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); - my $good_file = ''; - while (
) { - $_ =~ s/^\s+//; - my ($z,undef,$ff) = split /\s+/; - if (($ff)&&(-e $ff)) { - my $md_response = `md5sum $ff`; - if ($md_response =~ /$md5sum/) { - $good_file = (split /\s+/,$md_response)[1]; - } - } - } - close(DD); - if ($good_file eq '') { - die "No valid instance of key: $key"; - } - else { - print "cp $good_file $files[0]\n"; - if ($do_ops) { - system("cp $good_file $files[0]"); - } - } - # Begin my best validation thought - { - my $tmpmd5 = local_md5sum($files[0]); - if ($tmpmd5 ne $md5sum) { - die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; - } - } - # End my best validation thought. - - $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. - ## We've done work here, so we can't do a cull this iteration. - if ($do_cull == 1) { $do_cull = -1; } - } - ## We have more than zero bad copies. We may cull some of these in the future, but we should try to - ## leave everything in the best state possible. - printf(">> %d %d\n",$Ngood, $#files + 1); - if ($Ngood != $#files + 1) { - my $good_copy; - my $good_copy_index; - foreach my $site_key (keys %good_instances) { - if ($#{ $good_instances{$site_key} } != -1) { - $good_copy_index = $good_instances{$site_key}[0]; - $good_copy = $files[$good_copy_index]; - last; - } - } - printf(">> GOOD: $good_copy\n"); - foreach my $site_key (keys %bad_instances) { - foreach my $bad_copy_index (@{ $bad_instances{$site_key} }) { - print "cp $good_copy $files[$bad_copy_index]\n"; - if ($do_ops) { - system("cp $good_copy $files[$bad_copy_index]"); - } - my $tmpmd5 = local_md5sum($files[$bad_copy_index]); - if ($tmpmd5 ne $md5sum) { - ## This isn't super critical, so we don't need to die here. - warn "Post-repair md5sum does not match! $tmpmd5 != $md5sum: $files[$bad_copy_index]"; - } - else { - ## success - push @{ $good_instances{$site_key} }, $bad_copy_index; - } - } - } - ## We've done work here, so we can't do a cull this iteration. - if ($do_cull == 1) { $do_cull = -1; } - } - - ## We can now attempt to make replicated copies to the sites that require additional copies. - foreach my $site_key (keys %requirement_map) { - my $have_instances = $#{ $good_instances{$site_key} } + 1; - print "## $site_key $have_instances $requirement_map{$site_key}\n"; - if ($#{ $good_instances{$site_key} } + 1 < $requirement_map{$site_key}) { - my $rep_vol = get_random_site_volume($site_key); - print "neb-replicate --volume $rep_vol $key\n"; - if ($do_ops) { - $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; - if ($@) { die $@; } +# $validation[$i] = sprintf(" % 3d %d %32s %s %s %s\n", +# $instance_exists,$is_good,$instance_md5sum, +# $files[$i],$instance_host,$instance_volume); +# } + +# $time = time() - $timer_start; +# # object_id ext_id epoch available existing total timer +# printf("%s %s %s %d %d %d %d\n",@$stat[0],@$stat[1],@$stat[4],@$stat[6],$existing_copies,@$stat[7],$timer); +# # instance_exists is_good instance_md5sum file instance_host instance_volume +# my $val_string = join('',@validation); +# print "$val_string"; + +# # Decide what to do + +# if ($Ngood == 0) { +# # If we have no copies, we can't do anything. +# die "No valid instance of key: $key\n"; +# } +# ## We have more than zero bad copies. We may cull some of these in the future, but we should try to +# ## leave everything in the best state possible. +# printf(">> %d %d\n",$Ngood, $#files + 1); +# my $site_key; # Scope issues? + +# if ($Ngood != $#files + 1) { +# my $good_copy; +# my $good_copy_index; +# foreach $site_key (keys %good_instances) { +# if ($#{ $good_instances{$site_key} } != -1) { +# $good_copy_index = $good_instances{$site_key}[0]; +# $good_copy = $files[$good_copy_index]; +# last; +# } +# } +# printf(">> GOOD: $good_copy\n"); +# foreach $site_key (keys %bad_instances) { +# foreach my $bad_copy_index (@{ $bad_instances{$site_key} }) { +# print "cp $good_copy $files[$bad_copy_index]\n"; +# if ($do_ops) { +# system("cp $good_copy $files[$bad_copy_index]"); +# } +# my $tmpmd5 = local_md5sum($files[$bad_copy_index]); +# if ($tmpmd5 ne $md5sum) { +# ## This isn't super critical, so we don't need to die here. +# warn "Post-repair md5sum does not match! $tmpmd5 != $md5sum: $files[$bad_copy_index]"; +# } +# else { +# ## success +# push @{ $good_instances{$site_key} }, $bad_copy_index; +# } +# } +# } +# ## We've done work here, so we can't do a cull this iteration. +# if ($do_cull == 1) { $do_cull = -1; } +# } + +# ## We can now attempt to make replicated copies to the sites that require additional copies. +# foreach $site_key (keys %requirement_map) { +# my $have_instances = $#{ $good_instances{$site_key} } + 1; +# print "## $site_key $have_instances $requirement_map{$site_key}\n"; +# if ($#{ $good_instances{$site_key} } + 1 < $requirement_map{$site_key}) { +# my $rep_vol = get_random_site_volume($site_key); +# print "neb-replicate --volume $rep_vol $key\n"; +# if ($do_ops) { +# $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; +# if ($@) { die $@; } - # Begin my best validation thought - system("sync") == 0 or die "Couldn't sync?"; - my $uris = $neb->find_instances($key,$rep_vol); - @$uris = map {URI->new($_)->file if $_} @$uris; - my $tmpmd5 = local_md5sum(${ $uris }[0]); - - my $validation_str = sprintf("% 3d %d %32s %s %s %s", - -1,-1,$tmpmd5, - ${ $uris }[0],"repl",$rep_vol); - print - join("\n" . " " x 4, $validation_str), "\n"; +# # Begin my best validation thought +# system("sync") == 0 or die "Couldn't sync?"; +# my $uris = $neb->find_instances($key,$rep_vol); +# @$uris = map {URI->new($_)->file if $_} @$uris; +# my $tmpmd5 = local_md5sum(${ $uris }[0]); + +# my $validation_str = sprintf("% 3d %d %32s %s %s %s", +# -1,-1,$tmpmd5, +# ${ $uris }[0],"repl",$rep_vol); +# print +# join("\n" . " " x 4, $validation_str), "\n"; - if ($tmpmd5 ne $md5sum) { - die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; - } - # End my best validation thought. - } - ## We've done work here, so we can't do a cull this iteration. - if ($do_cull == 1) { $do_cull = -1; } - } - } +# if ($tmpmd5 ne $md5sum) { +# die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; +# } +# # End my best validation thought. +# } +# ## We've done work here, so we can't do a cull this iteration. +# if ($do_cull == 1) { $do_cull = -1; } +# } +# } - ## Do culls if that's what we were going to do. - if ($do_cull == -1) { - die "Cull option passed, but files were modified in the scan/repair/replicate phase. Not running cull!\n"; - } - elsif ($do_cull == 1) { - ## At this point, we should have no files in the bad_instances lists, because we've repaired them. - foreach my $site_key (keys %good_instances) { - if ($#{ $good_instances{$site_key} } + 1 > $requirement_map{$site_key}) { - for ($i = $requirement_map{$site_key}; $i <= $#{ $good_instances{$site_key} }; $i++) { - my $cull_index = ${ $good_instances{$site_key} }[$i]; - my ($instance_host,$instance_volume) = parse_volume($files[$cull_index]); - print "neb-cull --volume $instance_volume $key\n"; - if ($do_ops) { - # The tilde here is to force hard volumes. Don't touch it. - # Also: the 2 is a "minimum number of copies" restriction. Let's not be crazy here. - $neb->cull($key,"~${instance_volume}",2) or die "failed to cull a superfluous instance"; - if ($@) { die "$@"; } - } - } # End loop over extra instances - } # End check for sites with extra instances - } # End loop over sites. - } # End cull +# ## Do culls if that's what we were going to do. +# if ($do_cull == -1) { +# die "Cull option passed, but files were modified in the scan/repair/replicate phase. Not running cull!\n"; +# } +# elsif ($do_cull == 1) { +# ## At this point, we should have no files in the bad_instances lists, because we've repaired them. +# foreach my $site_key (keys %good_instances) { +# if ($#{ $good_instances{$site_key} } + 1 > $requirement_map{$site_key}) { +# for ($i = $requirement_map{$site_key}; $i <= $#{ $good_instances{$site_key} }; $i++) { +# my $cull_index = ${ $good_instances{$site_key} }[$i]; +# my ($instance_host,$instance_volume) = parse_volume($files[$cull_index]); +# print "neb-cull --volume $instance_volume $key\n"; +# if ($do_ops) { +# # The tilde here is to force hard volumes. Don't touch it. +# # Also: the 2 is a "minimum number of copies" restriction. Let's not be crazy here. +# $neb->cull($key,"~${instance_volume}",2) or die "failed to cull a superfluous instance"; +# if ($@) { die "$@"; } +# } +# } # End loop over extra instances +# } # End check for sites with extra instances +# } # End loop over sites. +# } # End cull } ## End loop over imfiles.