Index: trunk/ippScripts/scripts/rawcheck.pl
===================================================================
--- trunk/ippScripts/scripts/rawcheck.pl	(revision 39786)
+++ trunk/ippScripts/scripts/rawcheck.pl	(revision 39796)
@@ -22,5 +22,5 @@
 
 
-my ($server,$dbname,$exp_id);
+my ($server,$dbname,$exp_id,$do_cull);
 
 $server = $ENV{'NEB_SERVER'} unless $server;
@@ -30,5 +30,10 @@
     'dbname=s'       => \$dbname,
     'exp_id|x=s'     => \$exp_id,
+    'cull'           => \$do_cull,
 ) || pod2usage( 2 );
+
+unless(defined($do_cull)) {
+    $do_cull = 0;
+}
 
 # Option parsing
@@ -42,34 +47,12 @@
 
 # Global options:
+## Define the configuration.  Ideally, this would be retrieved from the nebulous
+## database, but there are some issues with that (such as grouping the b nodes 
+## into a less restrictive "offsite" location).
 
 my $do_ops = 1;
-my %host_mapping = (
-    'XY01' => 'ipp010','XY02' => 'ipp011','XY03' => 'ipp006','XY04' => 'ipp054',
-    'XY05' => 'ipp007','XY06' => 'ipp055','XY10' => 'ipp008','XY11' => 'ipp056',
-    'XY12' => 'ipp009','XY13' => 'ipp057','XY14' => 'ipp058','XY15' => 'ipp059',
-    'XY16' => 'ipp012','XY17' => 'ipp060','XY20' => 'ipp013','XY21' => 'ipp014',
-    'XY22' => 'ipp061','XY23' => 'ipp015','XY24' => 'ipp016','XY25' => 'ipp062',
-    'XY26' => 'ipp064','XY27' => 'ipp065','XY30' => 'ipp066','XY31' => 'ipp020',
-    'XY32' => 'ipp063','XY33' => 'ipp021','XY34' => 'ipp023','XY35' => 'ipp013',
-    'XY36' => 'ipp024','XY37' => 'ipp019','XY40' => 'ipp025','XY41' => 'ipp018',
-    'XY42' => 'ipp017','XY43' => 'ipp015','XY44' => 'ipp027','XY45' => 'ipp028',
-    'XY46' => 'ipp029','XY47' => 'ipp030','XY50' => 'ipp031','XY51' => 'ipp032',
-    'XY52' => 'ipp033','XY53' => 'ipp034','XY54' => 'ipp035','XY55' => 'ipp036',
-    'XY56' => 'ipp037','XY57' => 'ipp038','XY60' => 'ipp039','XY61' => 'ipp040',
-    'XY62' => 'ipp041','XY63' => 'ipp042','XY64' => 'ipp043','XY65' => 'ipp044',
-    'XY66' => 'ipp046','XY67' => 'ipp047','XY71' => 'ipp048','XY72' => 'ipp049',
-    'XY73' => 'ipp050','XY74' => 'ipp051','XY75' => 'ipp052','XY76' => 'ipp053');
-my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1,
-		    'ippb02' => 1, 'ippb03' => 1,
-		    'ippb04' => 1, 'ippb05' => 1,
-		    'ippb06' => 1
-    );
-my %backup_destinations = (# 'ippb04' => 1,  # full
-			   # 'ippb05' => 1,  # full
-			   'ippb06' => 1
-    );
-my $backup_Nvols = 3;
-
-# Set up nebulous db interface
+my $i;
+
+## Set up nebulous db interface, and pull processing information to consider.
 my $neb = Nebulous::Client->new(
     proxy => "$server",
@@ -78,8 +61,88 @@
     unless defined $neb;
 
-# Pull data from the gpc1 database
+## This new implementation is somewhat messy, but is more general and adaptable.
+## First, we set up a requirement mapping, explaining where we want copies, and
+## how many copies we want at each site.
+my %requirement_map = ();
+$requirement_map{ITC} = 1;
+$requirement_map{OFFSITE} = 1;
+$requirement_map{MRTCB} = 0;
+
+## Second, construct a list of volumes, mapped to their site location, using the
+## same site locations as in the requirement map.
+my %volume_map = ();
+for ($i = 4; $i <= 21; $i++) {
+    my $vol = sprintf("ipp%03d.0",$i);
+    $volume_map{$vol} = 'MRTCB';
+}
+for ($i = 23; $i <= 32; $i++) {
+    my $vol = sprintf("ipp%03d.0",$i);
+    $volume_map{$vol} = 'MRTCB';
+}
+for ($i = 54; $i <= 97; $i++) {
+    my $vol = sprintf("ipp%03d.0",$i);
+    $volume_map{$vol} = 'MRTCB';
+}
+for ($i = 100; $i <= 104; $i++) {
+    my $vol = sprintf("ipp%03d.0",$i);
+    $volume_map{$vol} = 'MRTCB';
+    $vol = sprintf("ipp%03d.1",$i);
+    $volume_map{$vol} = 'MRTCB';
+}
+for ($i = 105; $i <= 117; $i++) {
+    my $vol = sprintf("ipp%03d.0",$i);
+    $volume_map{$vol} = 'ITC';
+    $vol = sprintf("ipp%03d.1",$i);
+    $volume_map{$vol} = 'ITC';
+}    
+for ($i = 0; $i <= 6; $i++) {
+    if ($i == 6) { next; } # This isn't "offsite", it's with the rest of the maui cluster.
+    my $vol = sprintf("ippb%02d.0",$i);
+    $volume_map{$vol} = 'OFFSITE';
+    $vol = sprintf("ippb%02d.1",$i);
+    $volume_map{$vol} = 'OFFSITE';
+    $vol = sprintf("ippb%02d.2",$i);
+    $volume_map{$vol} = 'OFFSITE';
+}    
+
+## Next, get disk space values, and check which hosts are listed as available.
+my %acceptable_volume = ();
+my $mounts = $neb->mounts();
+foreach my $vol_row (@$mounts) {
+    my ($mount_point, $total, $used, $vol_id, $name, $host, $path, $allocate, $available, $xattr) = @{ $vol_row };
+    if (($allocate == 1)&&($available == 1)&&( $used / $total < 0.98)) {
+	$acceptable_volume{$name} = 1;
+    }
+    else {
+	print "## $name $allocate $available $used $total\n";
+	$acceptable_volume{$name} = 0;
+    }
+
+    if (($name eq 'ipp106.0')||($name eq 'ipp106.1')) {
+	$acceptable_volume{$name} = 0;
+    }    
+}
+
+## Finally, generate lists containing which volumes are located at which site.  
+## This allows us to randomly select a volume from the list for the site that 
+## we plan on replicating to.
+my %volume_lists = ();
+foreach my $vol_key (keys %requirement_map) {
+    @{ $volume_lists{$vol_key} } = grep { $acceptable_volume{$_} == 1 } (
+	grep { $volume_map{$_} eq $vol_key } (keys %volume_map)
+    );
+    print "$vol_key " . join(' ', @{ $volume_lists{$vol_key} }) . "\n";
+
+    if ($#{ $volume_lists{$vol_key} } == -1) {
+	die "No acceptable volume found for site $vol_key!\n";
+    }
+}
+
+# die;
+# Pull data from the gpc1 database about the exposure to consider
 my $verbose = 0;
 my $mdcParser = PS::IPP::Metadata::Config->new;
 
+print("$regtool -processedimfile -exp_id $exp_id -dbname $dbname");
 my $regtool_cmd = "$regtool -processedimfile -exp_id $exp_id -dbname $dbname";
 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
@@ -92,4 +155,8 @@
     &my_die("Unable to parse metadata from regtool -processedimfile", $exp_id);
 
+my $timer_start = time();
+my $timer = time();
+print "## rawcheck.pl: $exp_id $dbname $do_cull $do_ops $timer_start\n";
+# Loop over the imfiles of this exposure.
 foreach my $imfile (@$imfiles) {
     my $key        = $imfile->{uri};
@@ -100,10 +167,8 @@
 
     if (!(defined($hostname))) { $hostname = 'ipp004'; }
-
-#    if (($data_state ne 'full')||($data_state ne 'compressed')) { next; } ## skip things that aren't full.
-    
-# Do validation
-
-    # neb-stat level handling
+    $timer = time() - $timer_start;
+    print ("\n# $key $data_state $md5sum $hostname $class_id T: $timer\n");
+
+    ## Get instances, and do validation that they have the correct md5sums.
     my $stat = $neb->stat($key);
     die "nebulous key: $key not found" unless $stat;   
@@ -111,92 +176,55 @@
     die "no instances found" unless $instances;   
     
-    my $user_copies;
-    eval {
-	$user_copies = $neb->getxattr($key, "user.copies");
-    };
-    unless(defined($user_copies)) {
-	$user_copies = 1;
-    }
-
-    my @validation;
-    my %md5sum_uniq;
     my $existing_copies = 0;
-    
-    my @existance;
-    my @md5sums;
+    my $Ngood = 0;
+    my %good_instances = ();
+    my %bad_instances  = ();
+
     my @files = map {URI->new($_)->file if $_} @$instances;
-    my @diskvols;
-    my @diskhosts;
-    my @quality;
-    my $Ngood = 0;
-    my $quality_mask = 0;
+    my @validation = ();
 
     for (my $i = 0; $i <= $#files; $i++) {
+	my ($instance_exists,$instance_md5sum,$instance_host,$instance_volume,$instance_site, $is_good);
+	($instance_host,$instance_volume) = parse_volume($files[$i]);
+	$instance_site = $volume_map{$instance_volume};
+	$is_good = 0;
+
 	if (-e $files[$i]) {
-	    $existance[$i] = 1;
+	    $instance_exists = 1;
 	    $existing_copies++;
-	    $md5sums[$i] = local_md5sum($files[$i]);
-	    $md5sum_uniq{$md5sums[$i]} = 1;
-
+	    $instance_md5sum = local_md5sum($files[$i]);
+	    if ($instance_md5sum eq $md5sum) {
+		push @{ $good_instances{$instance_site} }, $i;
+		$is_good = 1;
+		$Ngood++;
+	    }		
+	    else {
+		push @{ $bad_instances{$instance_site} }, $i;
+	    }
 	}
 	else {
-	    $existance[$i] = 0;
-	    $md5sums[$i] = 'NON-EXISTANT';
-	    $md5sum_uniq{$md5sums[$i]} = 1;
-	}
-	($diskhosts[$i],$diskvols[$i]) = parse_volume($files[$i]);
-	$validation[$i] = sprintf("% 3d %32s %s %d %s %d",
-				  $existance[$i],
-				  $md5sums[$i],
-				  $files[$i],
-				  $md5sums[$i] eq $md5sum,
-				  $diskhosts[$i],$diskvols[$i]
-	    );
-
-	# Pre-parse decisions
-	if ($md5sums[$i] ne $md5sum) {
-	    $quality[$i] = 0;
-	}
-	elsif ($existance[$i] == 0) {
-	    $quality[$i] = 0;
-	}
-	elsif (is_backup_volume($diskhosts[$i])) {
-	    $quality[$i] = 1;
-	    $quality_mask = $quality_mask | 1;
-	    $Ngood++;
-	}
-	elsif ($diskhosts[$i] eq $host_mapping{$class_id}) {
-	    $quality[$i] = 1;
-	    $quality_mask = $quality_mask | 2;
-	    $Ngood++;
-	}
-	else {
-	    $quality[$i] = 2;
-	    $Ngood++;
-	}
-    }
-
-    print "\n$key $data_state $md5sum $hostname $class_id\n";
-    if (1) {
-
-	print
-	    "object id:             ", @$stat[0], "\n",
-	    "key:                   ", @$stat[1], "\n";
-	print 
-	    "epoch:                 ", @$stat[4], "\n";
-	print 
-	    "md5sum count:          ", scalar(keys %md5sum_uniq), "\n";
-	print 
-	    "requested instances:   ", $user_copies, "\n",
-	    "available instances:   ", @$stat[6], "\n",
-	    "existing instances:    ", $existing_copies, "\n",
-	    "total instances:       ", @$stat[7], "\n",
-	    "instance location:\n", " " x 4;
-	print
-	    join("\n" . " " x 4, @validation), "\n";
-    }
+	    $instance_exists = 0;
+	    $instance_md5sum = 'NON-EXISTANT';
+	    push @{ $bad_instances{$instance_site} }, $i;
+	}
+
+	$validation[$i] = sprintf("    % 3d %d %32s %s %s %s\n",
+				  $instance_exists,$is_good,$instance_md5sum,
+				  $files[$i],$instance_host,$instance_volume);
+    }
+
+    $timer = time() - $timer_start;
+    # object_id ext_id epoch available existing total timer
+    printf("%s %s %s %d %d %d %d\n",@$stat[0],@$stat[1],@$stat[4],@$stat[6],$existing_copies,@$stat[7],$timer);
+    # instance_exists is_good instance_md5sum file instance_host instance_volume
+    my $val_string = join('',@validation);
+    print "$val_string";
 
 
     # Decide what to do
+
+    ## This block attempts to find an out-of-nebulous instance with a good md5sum, 
+    ## and substitutes it for the 0-th listed instance.  This gives us one good
+    ## copy to work with.
     if ($Ngood == 0) {
 	# DO something to attempt to fix this.
@@ -220,7 +248,8 @@
 	}
 	else {
-	    $quality[0] = 1;
 	    print "cp $good_file $files[0]\n";
-	    system("cp $good_file $files[0]");
+	    if ($do_ops) {
+		system("cp $good_file $files[0]");
+	    }
 	}
         # Begin my best validation thought
@@ -234,37 +263,49 @@
 
 	$Ngood = 1;  # We now hand off this single valid instance object to be handled by the Ngood=1 case.
-    }
-    
-    if ($quality[0] == 0) { # The first instance is bad.
-	# But since we're here, and not up there, there must be at least one good copy.  Find it.
-	for (my $i = 0; $i <= $#md5sums; $i++) {
-	    if ($md5sums[$i] eq $md5sum) { # Found it.
-		print "cp $files[$i] $files[0]\n";
-		system("cp $files[$i] $files[0]");
-                # Begin my best validation thought
-		{
-		    my $tmpmd5 = local_md5sum($files[0]);
-		    if ($tmpmd5 ne $md5sum) { 
-			die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-		    }
+        ## We've done work here, so we can't do a cull this iteration.
+	if ($do_cull == 1) { $do_cull = -1; }
+    }
+
+    ## We have more than zero bad copies.  We may cull some of these in the future, but we should try to
+    ## leave everything in the best state possible.
+    printf(">> %d %d\n",$Ngood, $#files + 1);
+    if ($Ngood != $#files + 1) {
+	my $good_copy;
+	my $good_copy_index;
+	foreach my $site_key (keys %good_instances) {
+	    if ($#{ $good_instances{$site_key} } != -1) {
+		$good_copy_index = $good_instances{$site_key}[0];
+		$good_copy = $files[$good_copy_index];
+		last;
+	    }
+	}
+	printf(">> GOOD: $good_copy\n");
+	foreach my $site_key (keys %bad_instances) {
+	    foreach my $bad_copy_index (@{ $bad_instances{$site_key} }) {
+		print "cp $good_copy $files[$bad_copy_index]\n";
+		if ($do_ops) {
+		    system("cp $good_copy $files[$bad_copy_index]");
 		}
-                # End my best validation thought.
-		last; # We're done here now.
-	    }
-	}	
-    }
-
-    if ($Ngood == 1) { # We have only one version
-	if ($quality_mask & 1) { # ANd it's on a backup volume
-	    print "neb-replicate $key\n";
-	    if ($do_ops) {
-		$neb->replicate($key) or die "failed to replicate the single valid copy";
-		if ($@) { die $@; }
-		
-	    }
-	    
-	}
-	else { # And it's not, so put one there
-	    my $rep_vol = get_random_backup_volume();
+		my $tmpmd5 = local_md5sum($files[$bad_copy_index]);
+		if ($tmpmd5 ne $md5sum) { 
+		    ## This isn't super critical, so we don't need to die here.
+		    warn "Post-repair md5sum does not match! $tmpmd5 != $md5sum: $files[$bad_copy_index]";
+		}
+		else {
+		    ## success
+		    push @{ $good_instances{$site_key} }, $bad_copy_index;
+		}
+	    }
+	}
+	## We've done work here, so we can't do a cull this iteration.
+	if ($do_cull == 1) { $do_cull = -1; }
+    }
+
+    ## We can now attempt to make replicated copies to the sites that require additional copies.
+    foreach my $site_key (keys %requirement_map) {
+	my $have_instances = $#{ $good_instances{$site_key} } + 1;
+	print "## $site_key $have_instances $requirement_map{$site_key}\n";
+	if ($#{ $good_instances{$site_key} } + 1 < $requirement_map{$site_key}) {
+	    my $rep_vol = get_random_site_volume($site_key);
 	    print "neb-replicate --volume $rep_vol  $key\n";
 	    if ($do_ops) {
@@ -277,4 +318,11 @@
 		@$uris = map {URI->new($_)->file if $_} @$uris;
 		my $tmpmd5 = local_md5sum(${ $uris }[0]);
+
+		my $validation_str = sprintf("% 3d %d %32s %s %s %s",
+					  -1,-1,$tmpmd5,
+					  ${ $uris }[0],"repl",$rep_vol);
+		print
+		    join("\n" . " " x 4, $validation_str), "\n";
+		
 		if ($tmpmd5 ne $md5sum) { 
 		    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
@@ -282,52 +330,33 @@
                 # End my best validation thought.
 	    }	    
-	}
-    }
-    else { # N >= 2
-	if (!($quality_mask & 1)) { # no backup copy
-	    my $rep_vol = get_random_backup_volume();
-	    print "neb-replicate --volume $rep_vol  $key\n";
-	    if ($do_ops) {
-		$neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node";
-		if ($@) { die $@; }
-
-                # Begin my best validation thought
-		system("sync") == 0 or die "Couldn't sync?";
-		my $uris = $neb->find_instances($key,$rep_vol);
-		@$uris = map {URI->new($_)->file if $_} @$uris;
-		my $tmpmd5 = local_md5sum(${ $uris }[0]);
-		if ($tmpmd5 ne $md5sum) { 
-		    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-		}
-                # End my best validation thought.
-
-	    }	    
-	}
-	if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid.
-	    for (my $i = 0; $i <= $#files; $i++) {
-		if ($quality[$i] == 2) {
-		    $quality[$i] = 1;
-		    last;
-		}
-	    }
-	}
-	for (my $i = 0; $i <= $#files; $i++) {
-#	    print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n";
-	    if ($quality[$i] != 1) {
-		if ($existance[$i] == 0) { # This disk file doesn't exist.
-		    system("touch $files[$i]");
-		}
-		my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
-		print "neb-cull --volume $cull_vol $key\n";
-		if ($do_ops) {
-		    # The tilde here is to force hard volumes.  Don't touch it.
-		    $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
-		    if ($@) { die "$@"; }
-		}
-	    }
-	}
-    }
-#    exit(0);
-}
+	    ## We've done work here, so we can't do a cull this iteration.
+	    if ($do_cull == 1) { $do_cull = -1; }
+	}
+    }
+    
+    ## Do culls if that's what we were going to do.
+    if ($do_cull == -1) {
+	die "Cull option passed, but files were modified in the scan/repair/replicate phase.  Not running cull!\n";
+    }
+    elsif ($do_cull == 1) {
+	## At this point, we should have no files in the bad_instances lists, because we've repaired them.
+	foreach my $site_key (keys %good_instances) {
+	    if ($#{ $good_instances{$site_key} } + 1 > $requirement_map{$site_key}) {
+		for ($i = $requirement_map{$site_key}; $i <= $#{ $good_instances{$site_key} }; $i++) {
+		    my $cull_index = ${ $good_instances{$site_key} }[$i];
+		    my ($instance_host,$instance_volume) = parse_volume($files[$cull_index]);
+		    print "neb-cull --volume $instance_volume $key\n";
+		    if ($do_ops) {
+			# The tilde here is to force hard volumes.  Don't touch it.
+			# Also: the 2 is a "minimum number of copies" restriction.  Let's not be crazy here.
+			$neb->cull($key,"~${instance_volume}",2) or die "failed to cull a superfluous instance";
+			if ($@) { die "$@"; }
+		    }
+		} # End loop over extra instances
+	    } # End check for sites with extra instances
+	} # End loop over sites.
+    } # End cull
+
+} ## End loop over imfiles.
 
 sub local_md5sum {
@@ -350,22 +379,13 @@
     my $filename = shift(@_);
     my $full_volume   = (split /\//, $filename)[2];
-    my ($hostname,$vol_index) = split /\./, $full_volume; # /;
-    return($hostname,$vol_index);
-}
-
-sub is_backup_volume {
-    my $hostname = shift(@_);
-    if (exists($backup_hosts{$hostname})) {
-	return(1);
-    }
-    return(0);
-}
-
-sub get_random_backup_volume {
-    my $NN = scalar keys %backup_destinations;
-    my $backup_host = (keys %backup_destinations)[int(rand($NN))];
-    my $backup_vol  = int(rand($backup_Nvols));
-    
-    return("${backup_host}.${backup_vol}");
+    my ($hostname,undef) = split /\./, $full_volume; # /;
+    return($hostname,$full_volume);
+}
+
+sub get_random_site_volume {
+    my $site_key = shift(@_);
+    my $NN = scalar @{ $volume_lists{$site_key} };
+    my $backup_volume = ${ $volume_lists{$site_key} }[int(rand($NN))];
+    return($backup_volume);
 }