Index: trunk/ippScripts/scripts/permcheck.pl
===================================================================
--- trunk/ippScripts/scripts/permcheck.pl	(revision 36702)
+++ trunk/ippScripts/scripts/permcheck.pl	(revision 36702)
@@ -0,0 +1,414 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings FATAL => qw( all );
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use Nebulous::Client;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version );
+use Pod::Usage qw( pod2usage );
+use IPC::Cmd 0.36 qw( can_run run);
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+
+use Digest::MD5;
+use URI;
+
+my $missing_tools = 0;
+#my $regtool  = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
+my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
+my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
+my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
+
+my ($server,$dbname,$stage,$stage_id);
+
+$server = $ENV{'NEB_SERVER'} unless $server;
+# $dbname = 'gpc1'; 
+GetOptions(
+    'server|s=s'     => \$server,
+    'dbname=s'       => \$dbname,
+    'stage=s'        => \$stage,
+    'stage_id|x=s'     => \$stage_id,
+) || pod2usage( 2 );
+
+# Option parsing
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --server", -exitval => 2 )
+    unless $server;
+pod2usage( -msg => "Required options: --dbname", -exitval => 2 )
+    unless $dbname;
+pod2usage( -msg => "Required options: --stage [camera|stack|skycal]", -exitval => 2 )
+    unless $stage;
+pod2usage( -msg => "missing key", exitval => 2 )
+    unless defined $stage_id;
+
+# Global options:
+
+my $do_ops = 1;
+my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1,
+		    'ippb02' => 1, 'ippb03' => 1,
+		    'ippb04' => 1, 'ippb05' => 1
+    );
+my %backup_destinations = ('ippb04' => 1,
+			   'ippb05' => 1);
+my $backup_Nvols = 3;
+
+my $ipprc;
+if ($dbname eq 'gpc1') {
+    $ipprc = PS::IPP::Config->new( "GPC1" );
+}
+else {
+    die "Unknown camera to use.";
+}
+
+# Set up nebulous db interface
+my $neb = Nebulous::Client->new(
+    proxy => "$server",
+);
+die "can't connected to Nebulous Server: $server"
+    unless defined $neb;
+
+# Pull data from the gpc1 database
+my $verbose = 0;
+my $mdcParser = PS::IPP::Metadata::Config->new;
+
+my $files;
+if ($stage eq 'camera') {
+    my $cmd = "$camtool -processedexp -cam_id $stage_id -dbname $dbname";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+	run(command => $cmd, verbose => 0);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform stagetool: $error_code", $stage_id);
+    }
+    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from stagetool", $stage_id);
+}
+elsif ($stage eq 'stack') {
+    my $cmd = "$stacktool -sumskyfile -stack_id $stage_id -dbname $dbname";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+	run(command => $cmd, verbose => 0);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform stagetool: $error_code", $stage_id);
+    }
+    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from stagetool", $stage_id);
+}
+elsif ($stage eq 'skycal') {
+    my $cmd = "$staticskytool -skycalresult -skycal_id $stage_id -dbname $dbname";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+	run(command => $cmd, verbose => 0);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform stagetool: $error_code", $stage_id);
+    }
+    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from stagetool", $stage_id);
+}
+
+my %components = ('camera' => ['PSASTRO.OUTPUT'],
+		  'stack'  => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP',
+			       'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'],
+		  'skycal' => ['PSASTRO.OUTPUT.CMF']);
+
+
+foreach my $entry (@$files) {
+    my $path_base = $entry->{path_base};
+    my $data_state = $entry->{state};
+    my $hostname = $entry->{hostname};
+    foreach my $product (@{ $components{$stage} }) {
+	my $key = $ipprc->filename($product,$path_base);
+
+# Do validation
+
+	# neb-stat level handling
+	my $stat = $neb->stat($key);
+	die "nebulous key: $key not found" unless $stat;   
+	my $instances = $neb->find_instances($key, 'any');
+	die "no instances found" unless $instances;   
+	
+	my $user_copies;
+	eval {
+	    $user_copies = $neb->getxattr($key, "user.copies");
+	};
+	unless(defined($user_copies)) {
+	    $user_copies = 1;
+	}
+
+	my $md5sum;
+	my @validation;
+	my %md5sum_uniq;
+	my $existing_copies = 0;
+	
+	my @existance;
+	my @md5sums;
+	my @diskfiles = map {URI->new($_)->file if $_} @$instances;
+	my @diskvols;
+	my @diskhosts;
+	my @quality;
+	my $Ngood = 0;
+	my $quality_mask = 0;
+	
+	for (my $i = 0; $i <= $#diskfiles; $i++) {
+	    if (-e $diskfiles[$i]) {
+		$existance[$i] = 1;
+		$existing_copies++;
+		$md5sums[$i] = local_md5sum($diskfiles[$i]);
+		$md5sum_uniq{$md5sums[$i]} = 1;
+	    }
+	    else {
+		$existance[$i] = 0;
+		$md5sums[$i] = 'NON-EXISTANT';
+		$md5sum_uniq{$md5sums[$i]} = 1;
+	    }
+	    ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]);
+	    $validation[$i] = sprintf("% 3d %32s %s %s %d",
+				      $existance[$i],
+				      $md5sums[$i],
+				      $diskfiles[$i],
+				      $diskhosts[$i],$diskvols[$i]
+	    );
+	    
+	    # Pre-parse decisions
+	    if ($existance[$i] == 0) {
+		$quality[$i] = 0;
+	    }
+	    elsif (is_backup_volume($diskhosts[$i])) {
+		$quality[$i] = 1;
+		$quality_mask = $quality_mask | 1;
+		$Ngood++;
+	    }
+	    else {
+		$quality[$i] = 2;
+		$Ngood++;
+	    }
+	}
+	if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth.
+	    die "There are multiple md5sum values for $key";
+	}
+	$md5sum = (keys(%md5sum_uniq))[0];
+
+	print "\n$key $data_state $md5sum $hostname\n";
+	if (1) {
+	    
+	    print
+		"object id:             ", @$stat[0], "\n",
+		"key:                   ", @$stat[1], "\n";
+	    print 
+		"epoch:                 ", @$stat[4], "\n";
+	    print 
+		"md5sum count:          ", scalar(keys %md5sum_uniq), "\n";
+	    print 
+		"requested instances:   ", $user_copies, "\n",
+		"available instances:   ", @$stat[6], "\n",
+		"existing instances:    ", $existing_copies, "\n",
+		"total instances:       ", @$stat[7], "\n",
+		"instance location:\n", " " x 4;
+	    print
+		join("\n" . " " x 4, @validation), "\n";
+	}
+
+	# Decide what to do
+	if ($Ngood == 0) {
+	    # DO something to attempt to fix this.
+	    my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/;
+	    
+	    open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |");
+	    my $good_file = '';
+	    while (<DD>) {
+		$_ =~ s/^\s+//;
+		my ($z,undef,$ff) = split /\s+/;
+		if (($ff)&&(-e $ff)) {
+		    my $md_response = `md5sum $ff`;
+		    if ($md_response =~ /$md5sum/) {
+			$good_file = (split /\s+/,$md_response)[1];
+		    }
+		}
+	    }
+	    close(DD);
+	    if ($good_file eq '') {
+		die "No valid instance of key: $key";
+	    }
+	    else {
+		$quality[0] = 1;
+		print "cp $good_file $diskfiles[0]\n";
+		system("cp $good_file $diskfiles[0]");
+	    }
+	    # Begin my best validation thought
+	    {
+		my $tmpmd5 = local_md5sum($diskfiles[0]);
+		if ($tmpmd5 ne $md5sum) { 
+		    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+		}
+	    }
+	    # End my best validation thought.
+	    
+	    $Ngood = 1;  # We now hand off this single valid instance object to be handled by the Ngood=1 case.
+	}
+    
+	if ($quality[0] == 0) { # The first instance is bad.
+	    # But since we're here, and not up there, there must be at least one good copy.  Find it.
+	    for (my $i = 0; $i <= $#md5sums; $i++) {
+		if ($md5sums[$i] eq $md5sum) { # Found it.
+		    print "cp $diskfiles[$i] $diskfiles[0]\n";
+		    system("cp $diskfiles[$i] $diskfiles[0]");
+		    # Begin my best validation thought
+		    {
+			my $tmpmd5 = local_md5sum($diskfiles[0]);
+			if ($tmpmd5 ne $md5sum) { 
+			    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+			}
+		    }
+		    # End my best validation thought.
+		    last; # We're done here now.
+		}
+	    }	
+	}
+	
+	if ($Ngood == 1) { # We have only one version
+	    if ($quality_mask & 1) { # ANd it's on a backup volume
+		print "neb-replicate $key\n";
+		if ($do_ops) {
+		    $neb->replicate($key) or die "failed to replicate the single valid copy";
+		    if ($@) { die $@; }
+		}
+	    }
+	    else { # And it's not, so put one there
+		my $rep_vol = get_random_backup_volume();
+		print "neb-replicate --volume $rep_vol  $key\n";
+		if ($do_ops) {
+		    $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node";
+		    if ($@) { die $@; }
+		    
+		    # Begin my best validation thought
+		    system("sync") == 0 or die "Couldn't sync?";
+		    my $uris = $neb->find_instances($key,$rep_vol);
+		    @$uris = map {URI->new($_)->file if $_} @$uris;
+		    my $tmpmd5 = local_md5sum(${ $uris }[0]);
+		    if ($tmpmd5 ne $md5sum) { 
+			die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+		    }
+		    # End my best validation thought.
+		}	    
+	    }
+	}
+	else { # N >= 2
+	    if (!($quality_mask & 1)) { # no backup copy
+		my $rep_vol = get_random_backup_volume();
+		print "neb-replicate --volume $rep_vol  $key\n";
+		if ($do_ops) {
+		    $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node";
+		    if ($@) { die $@; }
+		    
+		    # Begin my best validation thought
+		    system("sync") == 0 or die "Couldn't sync?";
+		    my $uris = $neb->find_instances($key,$rep_vol);
+		    @$uris = map {URI->new($_)->file if $_} @$uris;
+		    my $tmpmd5 = local_md5sum(${ $uris }[0]);
+		    if ($tmpmd5 ne $md5sum) { 
+			die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+		    }
+		    # End my best validation thought.
+		}	    
+	    }
+	    if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid.
+		for (my $i = 0; $i <= $#diskfiles; $i++) {
+		    if ($quality[$i] == 2) {
+			$quality[$i] = 1;
+			last;
+		    }
+		}
+	    }
+	    for (my $i = 0; $i <= $#diskfiles; $i++) {
+#	    print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n";
+		if ($quality[$i] != 1) {
+		    if ($existance[$i] == 0) { # This disk file doesn't exist.
+			system("touch $diskfiles[$i]");
+		    }
+		    my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
+		    print "neb-cull --volume $cull_vol $key\n";
+		    if ($do_ops) {
+			# The tilde here is to force hard volumes.  Don't touch it.
+			$neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
+			if ($@) { die "$@"; }
+		    }
+		}
+	    }
+	}
+    } # end product for this entry
+} # end entry for this id
+
+sub local_md5sum {
+    my $filename = shift;
+    my $volume   = (split /\//, $filename)[2];
+    my $host     = $volume;
+    $host =~ s/\.\d//;
+#    print "$filename $host $volume\n";
+    my $response = `ssh $host md5sum $filename`;
+    chomp($response);
+    my ($sum, undef) = split /\s+/, $response;
+    unless(defined($sum)) {
+	my_die("Failed to calculate md5sum locally. $filename $host $volume");
+    }
+    return($sum);
+}
+
+
+sub parse_volume {
+    my $filename = shift(@_);
+    my $full_volume   = (split /\//, $filename)[2];
+    my ($hostname,$vol_index) = split /\./, $full_volume; # /;
+    return($hostname,$vol_index);
+}
+
+sub is_backup_volume {
+    my $hostname = shift(@_);
+    if (exists($backup_hosts{$hostname})) {
+	return(1);
+    }
+    return(0);
+}
+
+sub get_random_backup_volume {
+    my $NN = scalar keys %backup_destinations;
+    my $backup_host = (keys %backup_destinations)[int(rand($NN))];
+    my $backup_vol  = int(rand($backup_Nvols));
+    
+    return("${backup_host}.${backup_vol}");
+}
+
+sub my_die {
+    my $msg = shift(@_);
+    print $msg . "\n";
+    foreach my $a (@_) {
+	print "ARG: $a\n";
+    }
+    die;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+__END__
+
+