Index: /trunk/ippScripts/scripts/permcheck.pl
===================================================================
--- /trunk/ippScripts/scripts/permcheck.pl	(revision 37138)
+++ /trunk/ippScripts/scripts/permcheck.pl	(revision 37139)
@@ -51,8 +51,11 @@
 my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1,
 		    'ippb02' => 1, 'ippb03' => 1,
-		    'ippb04' => 1, 'ippb05' => 1
+		    'ippb04' => 1, 'ippb05' => 1,
+		    'ippb06' => 1
     );
-my %backup_destinations = ('ippb04' => 1,
-			   'ippb05' => 1);
+my %backup_destinations = (#'ippb04' => 1,
+			   #'ippb05' => 1,
+			   'ippb06' => 1
+);
 my $backup_Nvols = 3;
 
@@ -111,5 +114,5 @@
 }
 
-my %components = ('camera' => ['PSASTRO.OUTPUT'],
+my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'],
 		  'stack'  => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP',
 			       'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'],
@@ -122,222 +125,239 @@
     my $hostname = $entry->{hostname};
     foreach my $product (@{ $components{$stage} }) {
-	my $key = $ipprc->filename($product,$path_base);
-
+	my @keys = ();
+	if (($stage eq 'camera')&&($product eq 'PSASTRO.OUTPUT.MASK')) {
+	    my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06',
+			'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17',
+			'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27',
+			'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37',
+			'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47',
+			'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57',
+			'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67',
+			'XY71','XY72','XY73','XY74','XY75','XY76');
+	    foreach my $ota (@otas) {
+		push @keys, $ipprc->filename($product,$path_base,$ota);
+	    }	    
+	}
+	else {
+	    push @keys, $ipprc->filename($product,$path_base);
+	}
 # Do validation
-
-	# neb-stat level handling
-	my $stat = $neb->stat($key);
-	die "nebulous key: $key not found" unless $stat;   
-	my $instances = $neb->find_instances($key, 'any');
-	die "no instances found" unless $instances;   
 	
-	my $user_copies;
-	eval {
-	    $user_copies = $neb->getxattr($key, "user.copies");
-	};
-	unless(defined($user_copies)) {
-	    $user_copies = 1;
-	}
-
-	my $md5sum;
-	my @validation;
-	my %md5sum_uniq;
-	my $existing_copies = 0;
-	
-	my @existance;
-	my @md5sums;
-	my @diskfiles = map {URI->new($_)->file if $_} @$instances;
-	my @diskvols;
-	my @diskhosts;
-	my @quality;
-	my $Ngood = 0;
-	my $quality_mask = 0;
-	
-	for (my $i = 0; $i <= $#diskfiles; $i++) {
-	    if (-e $diskfiles[$i]) {
-		$existance[$i] = 1;
-		$existing_copies++;
-		$md5sums[$i] = local_md5sum($diskfiles[$i]);
-		$md5sum_uniq{$md5sums[$i]} = 1;
-	    }
-	    else {
-		$existance[$i] = 0;
-		$md5sums[$i] = 'NON-EXISTANT';
-		$md5sum_uniq{$md5sums[$i]} = 1;
-	    }
-	    ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]);
-	    $validation[$i] = sprintf("% 3d %32s %s %s %d",
-				      $existance[$i],
-				      $md5sums[$i],
-				      $diskfiles[$i],
-				      $diskhosts[$i],$diskvols[$i]
-	    );
-	    
-	    # Pre-parse decisions
-	    if ($existance[$i] == 0) {
-		$quality[$i] = 0;
-	    }
-	    elsif (is_backup_volume($diskhosts[$i])) {
-		$quality[$i] = 1;
-		$quality_mask = $quality_mask | 1;
-		$Ngood++;
-	    }
-	    else {
-		$quality[$i] = 2;
-		$Ngood++;
-	    }
-	}
-	if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth.
-	    die "There are multiple md5sum values for $key";
-	}
-	$md5sum = (keys(%md5sum_uniq))[0];
-
-	print "\n$key $data_state $md5sum $hostname\n";
-	if (1) {
-	    
-	    print
-		"object id:             ", @$stat[0], "\n",
-		"key:                   ", @$stat[1], "\n";
-	    print 
-		"epoch:                 ", @$stat[4], "\n";
-	    print 
-		"md5sum count:          ", scalar(keys %md5sum_uniq), "\n";
-	    print 
-		"requested instances:   ", $user_copies, "\n",
-		"available instances:   ", @$stat[6], "\n",
-		"existing instances:    ", $existing_copies, "\n",
-		"total instances:       ", @$stat[7], "\n",
-		"instance location:\n", " " x 4;
-	    print
-		join("\n" . " " x 4, @validation), "\n";
-	}
-
-	# Decide what to do
-	if ($Ngood == 0) {
-	    # DO something to attempt to fix this.
-	    my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/;
-	    
-	    open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |");
-	    my $good_file = '';
-	    while (<DD>) {
-		$_ =~ s/^\s+//;
-		my ($z,undef,$ff) = split /\s+/;
-		if (($ff)&&(-e $ff)) {
-		    my $md_response = `md5sum $ff`;
-		    if ($md_response =~ /$md5sum/) {
-			$good_file = (split /\s+/,$md_response)[1];
-		    }
-		}
-	    }
-	    close(DD);
-	    if ($good_file eq '') {
-		die "No valid instance of key: $key";
-	    }
-	    else {
-		$quality[0] = 1;
-		print "cp $good_file $diskfiles[0]\n";
-		system("cp $good_file $diskfiles[0]");
-	    }
-	    # Begin my best validation thought
-	    {
-		my $tmpmd5 = local_md5sum($diskfiles[0]);
-		if ($tmpmd5 ne $md5sum) { 
-		    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-		}
-	    }
-	    # End my best validation thought.
-	    
-	    $Ngood = 1;  # We now hand off this single valid instance object to be handled by the Ngood=1 case.
-	}
-    
-	if ($quality[0] == 0) { # The first instance is bad.
-	    # But since we're here, and not up there, there must be at least one good copy.  Find it.
-	    for (my $i = 0; $i <= $#md5sums; $i++) {
-		if ($md5sums[$i] eq $md5sum) { # Found it.
-		    print "cp $diskfiles[$i] $diskfiles[0]\n";
-		    system("cp $diskfiles[$i] $diskfiles[0]");
-		    # Begin my best validation thought
-		    {
-			my $tmpmd5 = local_md5sum($diskfiles[0]);
+	foreach my $key (@keys) {
+	    # neb-stat level handling
+	    my $stat = $neb->stat($key);
+	    die "nebulous key: $key not found" unless $stat;   
+	    my $instances = $neb->find_instances($key, 'any');
+	    die "no instances found" unless $instances;   
+	    
+	    my $user_copies;
+	    eval {
+		$user_copies = $neb->getxattr($key, "user.copies");
+	    };
+	    unless(defined($user_copies)) {
+		$user_copies = 1;
+	    }
+	    
+	    my $md5sum;
+	    my @validation;
+	    my %md5sum_uniq;
+	    my $existing_copies = 0;
+	    
+	    my @existance;
+	    my @md5sums;
+	    my @diskfiles = map {URI->new($_)->file if $_} @$instances;
+	    my @diskvols;
+	    my @diskhosts;
+	    my @quality;
+	    my $Ngood = 0;
+	    my $quality_mask = 0;
+	    
+	    for (my $i = 0; $i <= $#diskfiles; $i++) {
+		if (-e $diskfiles[$i]) {
+		    $existance[$i] = 1;
+		    $existing_copies++;
+		    $md5sums[$i] = local_md5sum($diskfiles[$i]);
+		    $md5sum_uniq{$md5sums[$i]} = 1;
+		}
+		else {
+		    $existance[$i] = 0;
+		    $md5sums[$i] = 'NON-EXISTANT';
+		    $md5sum_uniq{$md5sums[$i]} = 1;
+		}
+		($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]);
+		$validation[$i] = sprintf("% 3d %32s %s %s %d",
+					  $existance[$i],
+					  $md5sums[$i],
+					  $diskfiles[$i],
+					  $diskhosts[$i],$diskvols[$i]
+		    );
+		
+		# Pre-parse decisions
+		if ($existance[$i] == 0) {
+		    $quality[$i] = 0;
+		}
+		elsif (is_backup_volume($diskhosts[$i])) {
+		    $quality[$i] = 1;
+		    $quality_mask = $quality_mask | 1;
+		    $Ngood++;
+		}
+		else {
+		    $quality[$i] = 2;
+		    $Ngood++;
+		}
+	    }
+	    if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth.
+		die "There are multiple md5sum values for $key";
+	    }
+	    $md5sum = (keys(%md5sum_uniq))[0];
+	    
+	    print "\n$key $data_state $md5sum $hostname\n";
+	    if (1) {
+		
+		print
+		    "object id:             ", @$stat[0], "\n",
+		    "key:                   ", @$stat[1], "\n";
+		print 
+		    "epoch:                 ", @$stat[4], "\n";
+		print 
+		    "md5sum count:          ", scalar(keys %md5sum_uniq), "\n";
+		print 
+		    "requested instances:   ", $user_copies, "\n",
+		    "available instances:   ", @$stat[6], "\n",
+		    "existing instances:    ", $existing_copies, "\n",
+		    "total instances:       ", @$stat[7], "\n",
+		    "instance location:\n", " " x 4;
+		print
+		    join("\n" . " " x 4, @validation), "\n";
+	    }
+	    
+	    # Decide what to do
+	    if ($Ngood == 0) {
+		# DO something to attempt to fix this.
+		my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/;
+		
+		open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |");
+		my $good_file = '';
+		while (<DD>) {
+		    $_ =~ s/^\s+//;
+		    my ($z,undef,$ff) = split /\s+/;
+		    if (($ff)&&(-e $ff)) {
+			my $md_response = `md5sum $ff`;
+			if ($md_response =~ /$md5sum/) {
+			    $good_file = (split /\s+/,$md_response)[1];
+			}
+		    }
+		}
+		close(DD);
+		if ($good_file eq '') {
+		    die "No valid instance of key: $key";
+		}
+		else {
+		    $quality[0] = 1;
+		    print "cp $good_file $diskfiles[0]\n";
+		    system("cp $good_file $diskfiles[0]");
+		}
+		# Begin my best validation thought
+		{
+		    my $tmpmd5 = local_md5sum($diskfiles[0]);
+		    if ($tmpmd5 ne $md5sum) { 
+			die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+		    }
+		}
+		# End my best validation thought.
+		
+		$Ngood = 1;  # We now hand off this single valid instance object to be handled by the Ngood=1 case.
+	    }
+	    
+	    if ($quality[0] == 0) { # The first instance is bad.
+		# But since we're here, and not up there, there must be at least one good copy.  Find it.
+		for (my $i = 0; $i <= $#md5sums; $i++) {
+		    if ($md5sums[$i] eq $md5sum) { # Found it.
+			print "cp $diskfiles[$i] $diskfiles[0]\n";
+			system("cp $diskfiles[$i] $diskfiles[0]");
+			# Begin my best validation thought
+			{
+			    my $tmpmd5 = local_md5sum($diskfiles[0]);
+			    if ($tmpmd5 ne $md5sum) { 
+				die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+			    }
+			}
+			# End my best validation thought.
+			last; # We're done here now.
+		    }
+		}	
+	    }
+	    
+	    if ($Ngood == 1) { # We have only one version
+		if ($quality_mask & 1) { # ANd it's on a backup volume
+		    print "neb-replicate $key\n";
+		    if ($do_ops) {
+			$neb->replicate($key) or die "failed to replicate the single valid copy";
+			if ($@) { die $@; }
+		    }
+		}
+		else { # And it's not, so put one there
+		    my $rep_vol = get_random_backup_volume();
+		    print "neb-replicate --volume $rep_vol  $key\n";
+		    if ($do_ops) {
+			$neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node";
+			if ($@) { die $@; }
+			
+			# Begin my best validation thought
+			system("sync") == 0 or die "Couldn't sync?";
+			my $uris = $neb->find_instances($key,$rep_vol);
+			@$uris = map {URI->new($_)->file if $_} @$uris;
+			my $tmpmd5 = local_md5sum(${ $uris }[0]);
 			if ($tmpmd5 ne $md5sum) { 
 			    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
 			}
-		    }
-		    # End my best validation thought.
-		    last; # We're done here now.
-		}
-	    }	
-	}
-	
-	if ($Ngood == 1) { # We have only one version
-	    if ($quality_mask & 1) { # ANd it's on a backup volume
-		print "neb-replicate $key\n";
-		if ($do_ops) {
-		    $neb->replicate($key) or die "failed to replicate the single valid copy";
-		    if ($@) { die $@; }
-		}
-	    }
-	    else { # And it's not, so put one there
-		my $rep_vol = get_random_backup_volume();
-		print "neb-replicate --volume $rep_vol  $key\n";
-		if ($do_ops) {
-		    $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node";
-		    if ($@) { die $@; }
-		    
-		    # Begin my best validation thought
-		    system("sync") == 0 or die "Couldn't sync?";
-		    my $uris = $neb->find_instances($key,$rep_vol);
-		    @$uris = map {URI->new($_)->file if $_} @$uris;
-		    my $tmpmd5 = local_md5sum(${ $uris }[0]);
-		    if ($tmpmd5 ne $md5sum) { 
-			die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-		    }
-		    # End my best validation thought.
-		}	    
-	    }
-	}
-	else { # N >= 2
-	    if (!($quality_mask & 1)) { # no backup copy
-		my $rep_vol = get_random_backup_volume();
-		print "neb-replicate --volume $rep_vol  $key\n";
-		if ($do_ops) {
-		    $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node";
-		    if ($@) { die $@; }
-		    
-		    # Begin my best validation thought
-		    system("sync") == 0 or die "Couldn't sync?";
-		    my $uris = $neb->find_instances($key,$rep_vol);
-		    @$uris = map {URI->new($_)->file if $_} @$uris;
-		    my $tmpmd5 = local_md5sum(${ $uris }[0]);
-		    if ($tmpmd5 ne $md5sum) { 
-			die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-		    }
-		    # End my best validation thought.
-		}	    
-	    }
-	    if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid.
+			# End my best validation thought.
+		    }	    
+		}
+	    }
+	    else { # N >= 2
+		if (!($quality_mask & 1)) { # no backup copy
+		    my $rep_vol = get_random_backup_volume();
+		    print "neb-replicate --volume $rep_vol  $key\n";
+		    if ($do_ops) {
+			$neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node";
+			if ($@) { die $@; }
+			
+			# Begin my best validation thought
+			system("sync") == 0 or die "Couldn't sync?";
+			my $uris = $neb->find_instances($key,$rep_vol);
+			@$uris = map {URI->new($_)->file if $_} @$uris;
+			my $tmpmd5 = local_md5sum(${ $uris }[0]);
+			if ($tmpmd5 ne $md5sum) { 
+			    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+			}
+			# End my best validation thought.
+		    }	    
+		}
+		if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid.
+		    for (my $i = 0; $i <= $#diskfiles; $i++) {
+			if ($quality[$i] == 2) {
+			    $quality[$i] = 1;
+			    last;
+			}
+		    }
+		}
 		for (my $i = 0; $i <= $#diskfiles; $i++) {
-		    if ($quality[$i] == 2) {
-			$quality[$i] = 1;
-			last;
-		    }
-		}
-	    }
-	    for (my $i = 0; $i <= $#diskfiles; $i++) {
 #	    print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n";
-		if ($quality[$i] != 1) {
-		    if ($existance[$i] == 0) { # This disk file doesn't exist.
-			system("touch $diskfiles[$i]");
-		    }
-		    my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
-		    print "neb-cull --volume $cull_vol $key\n";
-		    if ($do_ops) {
-			# The tilde here is to force hard volumes.  Don't touch it.
-			$neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
-			if ($@) { die "$@"; }
-		    }
-		}
-	    }
-	}
+		    if ($quality[$i] != 1) {
+			if ($existance[$i] == 0) { # This disk file doesn't exist.
+			    system("touch $diskfiles[$i]");
+			}
+			my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
+			print "neb-cull --volume $cull_vol $key\n";
+			if ($do_ops) {
+			    # The tilde here is to force hard volumes.  Don't touch it.
+			    $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
+			    if ($@) { die "$@"; }
+			}
+		    }
+		}
+	    }
+	} # end keys for this product
     } # end product for this entry
 } # end entry for this id