Changeset 39924 for branches/czw_branch/20160809/ippScripts/scripts/permcheck.pl

Timestamp:

Jan 6, 2017, 11:18:12 AM (10 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/czw_branch/20160809

Files:

• . (modified) (1 prop)
• ippScripts/scripts/permcheck.pl (modified) (9 diffs)

branches/czw_branch/20160809/ippScripts/scripts/permcheck.pl

@@ -20 +20 @@
 my $missing_tools = 0;
 #my $regtool  = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
+my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
 my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
 my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
 my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
 my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
-
-my ($server,$dbname,$stage,$stage_id);
+my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1);
+
+my ($server,$dbname,$stage,$stage_id,$do_cull,$save_log);
 
 $server = $ENV{'NEB_SERVER'} unless $server;
@@ -34 +36 @@
     'stage=s'        => \$stage,
     'stage_id|x=s'     => \$stage_id,
+    'save_log'       => \$save_log,
+    'cull'           => \$do_cull,
 ) || pod2usage( 2 );
+
+unless(defined($do_cull)) {
+    $do_cull = 0;
+}
 
 # Option parsing
@@ -47 +55 @@
     unless defined $stage_id;
 
+if ($save_log) {
+    my $time = time();
+    my $logDest = "neb://any/perm_check/${stage}/${stage}_${stage_id}.${time}";
+    my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n";
+    $ipprc->redirect_output($logDest) or die "Could not redirect output to logfile ${logDest}\n";
+}
+
+
 # Global options:
-
+## Define the configuration.  Ideally, this would be retrieved from the nebulous                                                      
+## database, but there are some issues with that (such as grouping the b nodes                                                        
+## into a less restrictive "offsite" location).                                                                                       
 my $do_ops = 1;
-my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1,
-                    'ippb02' => 1, 'ippb03' => 1,
-                    'ippb04' => 1, 'ippb05' => 1,
-                    'ippb06' => 1
-    );
-my %backup_destinations = (#'ippb04' => 1,
-                           #'ippb05' => 1,
-                           'ippb06' => 1
-);
-
-if ($stage eq 'warp') { # These things are destined to have their only copy on the stsciXX nodes.
-    %backup_hosts = ('stsci00' => 1, 'stsci01' => 1, 'stsci02' => 1,
-                     #'stsci03' => 1, 
-                     'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1,
-                     'stsci07' => 1, 'stsci08' => 1, 'stsci09' => 1,
-                     'stsci10' => 1, 'stsci11' => 1, 'stsci12' => 1,
-                     'stsci13' => 1, 'stsci14' => 1, 'stsci15' => 1, 'stsci16' => 1,
-                     'stsci17' => 1, 'stsci18' => 1, 'stsci19' => 1);
-    %backup_destinations = %backup_hosts;
-}
-
-my $backup_Nvols = 3;
-
-my $ipprc;
-if ($dbname eq 'gpc1') {
-    $ipprc = PS::IPP::Config->new( "GPC1" );
-}
-else {
-    die "Unknown camera to use.";
-}
+my $i;
 
 # Set up nebulous db interface
@@ -88 +77 @@
     unless defined $neb;
 
+## This new implementation is somewhat messy, but is more general and adaptable.                                                      
+## First, we set up a requirement mapping, explaining where we want copies, and                                                       
+## how many copies we want at each site.                                                                                              
+my %requirement_map = ();
+$requirement_map{ITC} = 1;
+$requirement_map{OFFSITE} = 1;
+$requirement_map{MRTCB} = 0;
+
+## Second, construct a list of volumes, mapped to their site location, using the                                                      
+## same site locations as in the requirement map.                                                                                     
+my %volume_map = ();
+for ($i = 4; $i <= 21; $i++) {
+    my $vol = sprintf("ipp%03d.0",$i);
+    $volume_map{$vol} = 'MRTCB';
+}
+for ($i = 23; $i <= 32; $i++) {
+    my $vol = sprintf("ipp%03d.0",$i);
+    $volume_map{$vol} = 'MRTCB';
+}
+for ($i = 54; $i <= 97; $i++) {
+    my $vol = sprintf("ipp%03d.0",$i);
+    $volume_map{$vol} = 'MRTCB';
+}
+for ($i = 100; $i <= 104; $i++) {
+    my $vol = sprintf("ipp%03d.0",$i);
+    $volume_map{$vol} = 'MRTCB';
+    $vol = sprintf("ipp%03d.1",$i);
+    $volume_map{$vol} = 'MRTCB';
+}
+for ($i = 105; $i <= 117; $i++) {
+#    if ($i == 115) { next; }                                                                                                         
+    my $vol = sprintf("ipp%03d.0",$i);
+    $volume_map{$vol} = 'ITC';
+    $vol = sprintf("ipp%03d.1",$i);
+    $volume_map{$vol} = 'ITC';
+}
+for ($i = 0; $i <= 15; $i++) {
+    if ($i == 6) { next; } # This isn't "offsite", it's with the rest of the maui cluster.                                            
+    if ($i == 9) { next; } # Not online                                                                                               
+
+    my $vol = sprintf("ippb%02d.0",$i);
+    $volume_map{$vol} = 'OFFSITE';
+    $vol = sprintf("ippb%02d.1",$i);
+    $volume_map{$vol} = 'OFFSITE';
+    $vol = sprintf("ippb%02d.2",$i);
+    $volume_map{$vol} = 'OFFSITE';
+}
+
+## Next, get disk space values, and check which hosts are listed as available.
+my %acceptable_volume = ();
+my $mounts = $neb->mounts();
+foreach my $vol_row (@$mounts) {
+    my ($mount_point, $total, $used, $vol_id, $name, $host, $path, $allocate, $available, $xattr) = @{ $vol_row };
+    if (($allocate == 1)&&($available == 1)&&( $used / $total < 0.98)) {
+        $acceptable_volume{$name} = 1;
+    }
+    else {
+        print "## $name $allocate $available $used $total\n";
+        $acceptable_volume{$name} = 0;
+    }
+    if ($name =~ /ippb05/) {
+        $acceptable_volume{$name} = 0;
+    }
+#    if (($name eq 'ipp106.0')||($name eq 'ipp106.1')) {                                                                              
+#       $acceptable_volume{$name} = 0;                                                                                                
+#    }                                                                                                                                
+}
+
+## Finally, generate lists containing which volumes are located at which site.                                                        
+## This allows us to randomly select a volume from the list for the site that                                                         
+## we plan on replicating to.                                                                                                         
+my %volume_lists = ();
+foreach my $vol_key (keys %requirement_map) {
+    @{ $volume_lists{$vol_key} } = grep { $acceptable_volume{$_} == 1 } (
+        grep { $volume_map{$_} eq $vol_key } (keys %volume_map)
+    );
+    print "$vol_key " . join(' ', @{ $volume_lists{$vol_key} }) . "\n";
+
+    if ($#{ $volume_lists{$vol_key} } == -1) {
+        die "No acceptable volume found for site $vol_key!\n";
+    }
+}
+
 # Pull data from the gpc1 database
 my $verbose = 0;
 my $mdcParser = PS::IPP::Metadata::Config->new;
 
-my $files;
+# Not technically imfiles, but the nebulous check block already uses files as a variable.
+my $imfiles;
 if ($stage eq 'camera') {
     my $cmd = "$camtool -processedexp -cam_id $stage_id -dbname $dbname";
@@ -101 +174 @@
         &my_die("Unable to perform stagetool: $error_code", $stage_id);
     }
-    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
+    $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata from stagetool", $stage_id);
 }
@@ -112 +185 @@
         &my_die("Unable to perform stagetool: $error_code", $stage_id);
     }
-    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
+    $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata from stagetool", $stage_id);
 }
@@ -123 +196 @@
         &my_die("Unable to perform stagetool: $error_code", $stage_id);
     }
-    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
+    $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata from stagetool", $stage_id);
 }
@@ -134 +207 @@
         &my_die("Unable to perform stagetool: $error_code", $stage_id);
     }
-    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
+    $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata from stagetool", $stage_id);
 }
-
-my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'],
+elsif ($stage eq 'ff') {
+    my $cmd = "$fftool -result --ff_id $stage_id -dbname $dbname";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+        run(command => $cmd, verbose => 0);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform stagetool: $error_code", $stage_id);
+    }
+    $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata from stagetool", $stage_id);
+}
+
+
+my $timer_start = time();
+my $timer = time();
+my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK','PSPHOT.BACKMDL','PSPHOT.PSF.RAW.SAVE'],
                   'warp'   => ['PSWARP.OUTPUT','PSWARP.OUTPUT.MASK','PSWARP.OUTPUT.VARIANCE'],
                   'stack'  => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP',
                                'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'],
-                  'skycal' => ['PSASTRO.OUTPUT.CMF']);
-
-
-foreach my $entry (@$files) {
+                  'skycal' => ['PSASTRO.OUTPUT.CMF','PSPHOT.OUTPUT.CFF'],
+                  'ff'     => ['PSPHOT.OUT.CMF.MEF','PSPHOT.OUTPUT.CFF','PSPHOT.FULLFORCE.OUTPUT']);
+
+my $timer_start = time();
+my $timer = time();
+
+print "## permcheck.pl: $stage $stage_id $dbname $do_cull $do_ops $timer_start\n";
+foreach my $entry (@$imfiles) {
     my $path_base = $entry->{path_base};
     my $data_state = $entry->{state};
     my $hostname = $entry->{hostname};
     my $quality  = $entry->{quality};
+
+    $timer = $time() - $timer_start;
+    print "# $path_base $data_state $hostname $quality T: $timer\n";
     if ($quality != 0) { next; }
-    print "# $path_base $data_state $hostname $quality\n";
+
+    my @keys = ();
     foreach my $product (@{ $components{$stage} }) {
-        my @keys = ();
-        if (($stage eq 'camera')&&($product eq 'PSASTRO.OUTPUT.MASK')) {
-            my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06',
-                        'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17',
-                        'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27',
-                        'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37',
-                        'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47',
-                        'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57',
-                        'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67',
-                        'XY71','XY72','XY73','XY74','XY75','XY76');
-            foreach my $ota (@otas) {
-                push @keys, $ipprc->filename($product,$path_base,$ota);
-            }       
-        }
+        if ($stage eq 'camera') {
+            if ($product eq 'PSASTRO.OUTPUT.MASK') {
+                my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06',
+                            'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17',
+                            'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27',
+                            'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37',
+                            'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47',
+                            'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57',
+                            'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67',
+                            'XY71','XY72','XY73','XY74','XY75','XY76');
+                foreach my $ota (@otas) {
+                    push @keys, $ipprc->filename($product,$path_base,$ota);
+                }           
+            }
+            elsif (($product eq 'PSPHOT.BACKMDL')||($product eq 'PSPHOT.PSF.RAW.SAVE')) {
+                my $chip_id = $entry->{chip_id};
+                my $chip_cmd = "$chiptool -processedimfile -chip_id $chip_id -dbname $dbname";
+                my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+                    run(command => $chip_cmd, verbose => 0);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform stagetool: $error_code", $stage_id);
+                }
+                my $chip_$files = $mdcParser->parse_list(join "", @$stdout_buf) or
+                    &my_die("Unable to parse metadata from stagetool", $stage_id);
+                
+                foreach my $chip_entry (@$chip_files) {
+                    my $chip_path_base = $chip_entry->{path_base};
+                    my $class_id       = $chip_entry->{class_id};
+                    push @keys, $ipprc->filename($product,$chip_path_base,$class_id);
+                }
+            }
+        }
+        elsif (($stage eq 'ff')&&($product eq 'PSPHOT.FULLFORCE.OUTPUT')) {
+            my $ffsum_cmd = "$fftool -summary -ff_id $stage_id -dbname $dbname";
+            my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+                run(command => $ffsum_cmd, verbose => 0);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform stagetool: $error_code", $stage_id);
+            }
+            my $chip_$files = $mdcParser->parse_list(join "", @$stdout_buf) or
+                &my_die("Unable to parse metadata from stagetool", $stage_id);
+            
+            foreach my $chip_entry (@$chip_files) {
+                my $chip_path_base = $chip_entry->{path_base};
+                my $class_id       = $chip_entry->{class_id};
+                push @keys, $ipprc->filename($product,$chip_path_base,);
+            }
+        }           
         else {
             push @keys, $ipprc->filename($product,$path_base);
         }
 # Do validation
+    }
+
+    $timer = time() - $timer_start;
+    
+    printf("# Identified %d : %s\n",$#keys + 1, $timer);
+    foreach my $key (@keys) {
+        # neb-stat level handling
+        my $stat = $neb->stat($key);
+        die "nebulous key: $key not found" unless $stat;   
+        my $instances;
+        my $md5sum = '';
+
+        # This needs to be in an eval, because although we expect things to exist, they
+        # may not.  This is a fatal error in rawcheck, but need not be here.
+        eval {
+            $instances = $neb->find_instances($key, 'any');
+        };
+        unless (defined($instances)) { print "## skipping due to zero instances\n"; next; }
+        die "no instances found" unless $instances;   
         
-        foreach my $key (@keys) {
-            # neb-stat level handling
-            my $stat = $neb->stat($key);
-            die "nebulous key: $key not found" unless $stat;   
-            my $instances;
-            eval {
-                $instances = $neb->find_instances($key, 'any');
-            };
-            unless (defined($instances)) { print "## skipping due to zero instances\n"; next; }
-            die "no instances found" unless $instances;   
+        my $Ngood;
+        my %good_instances = ();
+        my %bad_instances  = ();
+
+        my @files = map {URI->new($_)->file if $_} @$instances;
+        my @validation = ();
+
+        for (my $i = 0; $i <= $#files; $i++) {
+            my ($instance_exists,$instance_md5sum,$instance_host,$instance_volume,$instance_site, $is_good);
+            ($instance_host,$instance_volume) = parse_volume($files[$i]);
+            $instance_site = $volume_map{$instance_volume};
+            $is_good = 0;
+
+            if (-e $files[$i]) {
+                $instance_exists = 1;
+                $existing_copies++;
+                $instance_md5sum = local_md5sum($files[$i]);
+
+                # This is bad, but I don't know what the right solution is.  We don't have 
+                # the md5sum a priori.  I think this is also the only major change needed 
+                # from rawcheck.pl
+                if (($md5sum eq '')&&($instance_md5sum ne 'd41d8cd98f00b204e9800998ecf8427e')) {
+                    $md5sum = $instance_md5sum;
+                }
+
+                if ($instance_md5sum eq $md5sum) {
+                    push @{ $good_instance{$instance_site} }, $i;
+                    $is_good = 1;
+                    $Ngood++;
+                }
+                else {
+                    push @{ $bad_instances{$instance_site} }, $i;
+                }
+            }
+            else {
+                $instance_exists = 0;
+                $instance_md5sum = 'NON-EXISTANT';
+                push @{ $bad_instances{$instance_site} }, $i;
+            }
             
-            my $user_copies;
-            eval {
-                $user_copies = $neb->getxattr($key, "user.copies");
-            };
-            unless(defined($user_copies)) {
-                $user_copies = 1;
-            }
-            
-            my $md5sum;
-            my @validation;
-            my %md5sum_uniq;
-            my $existing_copies = 0;
-            
-            my @existance;
-            my @md5sums;
-            my @diskfiles = map {URI->new($_)->file if $_} @$instances;
-            my @diskvols;
-            my @diskhosts;
-            my @quality;
-            my $Ngood = 0;
-            my $quality_mask = 0;
-            
-            for (my $i = 0; $i <= $#diskfiles; $i++) {
-                if (-e $diskfiles[$i]) {
-                    $existance[$i] = 1;
-                    $existing_copies++;
-                    $md5sums[$i] = local_md5sum($diskfiles[$i]);
-                    $md5sum_uniq{$md5sums[$i]} = 1;
-                }
-                else {
-                    $existance[$i] = 0;
-                    $md5sums[$i] = 'NON-EXISTANT';
-                    $md5sum_uniq{$md5sums[$i]} = 1;
-                }
-                ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]);
-                $validation[$i] = sprintf("% 3d %32s %s %s %d",
-                                          $existance[$i],
-                                          $md5sums[$i],
-                                          $diskfiles[$i],
-                                          $diskhosts[$i],$diskvols[$i]
-                    );
-                
-                # Pre-parse decisions
-                if ($existance[$i] == 0) {
-                    $quality[$i] = 0;
-                }
-                elsif (is_backup_volume($diskhosts[$i])) {
-                    $quality[$i] = 1;
-                    $quality_mask = $quality_mask | 1;
-                    $Ngood++;
-                }
-                else {
-                    $quality[$i] = 2;
-                    $Ngood++;
-                }
-            }
-            if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth.
-                die "There are multiple md5sum values for $key";
-            }
-            $md5sum = (keys(%md5sum_uniq))[0];
-            
-            print "\n$key $data_state $md5sum $hostname\n";
-            if (1) {
-                
-                print
-                    "object id:             ", @$stat[0], "\n",
-                    "key:                   ", @$stat[1], "\n";
-                print 
-                    "epoch:                 ", @$stat[4], "\n";
-                print 
-                    "md5sum count:          ", scalar(keys %md5sum_uniq), "\n";
-                print 
-                    "requested instances:   ", $user_copies, "\n",
-                    "available instances:   ", @$stat[6], "\n",
-                    "existing instances:    ", $existing_copies, "\n",
-                    "total instances:       ", @$stat[7], "\n",
-                    "instance location:\n", " " x 4;
-                print
-                    join("\n" . " " x 4, @validation), "\n";
-            }
-            
+            $validation[$i] = sprintf("    % 3d %d %32s %s %s %s\n",
+                                      $instance_exists,$is_good,$instance_md5sum,
+                                      $files[$i],$instance_host,$instance_volume);
+        }
+
+        $time = time() - $timer_start;
+        # object_id ext_id epoch available existing total timer
+        printf("%s %s %s %d %d %d %d\n",@$stat[0],@$stat[1],@$stat[4],@$stat[6],$existing_copies,@$stat[7],$timer);
+        # instance_exists is_good instance_md5sum file instance_host instance_volume
+        my $val_string = join('',@validation);
+        print "$val_string";
+
             # Decide what to do
-            if ($Ngood == 0) {
-                # DO something to attempt to fix this.
-                my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/;
-                
-                open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |");
-                my $good_file = '';
-                while (<DD>) {
-                    $_ =~ s/^\s+//;
-                    my ($z,undef,$ff) = split /\s+/;
-                    if (($ff)&&(-e $ff)) {
-                        my $md_response = `md5sum $ff`;
-                        if ($md_response =~ /$md5sum/) {
-                            $good_file = (split /\s+/,$md_response)[1];
-                        }
+
+    ## This block attempts to find an out-of-nebulous instance with a good md5sum, 
+    ## and substitutes it for the 0-th listed instance.  This gives us one good
+    ## copy to work with.
+        if ($Ngood == 0) {
+        # DO something to attempt to fix this.
+            my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/;
+
+            open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |");
+            my $good_file = '';
+            while (<DD>) {
+                $_ =~ s/^\s+//;
+                my ($z,undef,$ff) = split /\s+/;
+                if (($ff)&&(-e $ff)) {
+                    my $md_response = `md5sum $ff`;
+                    if ($md_response =~ /$md5sum/) {
+                        $good_file = (split /\s+/,$md_response)[1];
                     }
                 }
-                close(DD);
-                if ($good_file eq '') {
-                    die "No valid instance of key: $key";
-                }
-                else {
-                    $quality[0] = 1;
-                    print "cp $good_file $diskfiles[0]\n";
-                    vsystem("cp $good_file $diskfiles[0]");
-                }
-                # Begin my best validation thought
-                {
-                    my $tmpmd5 = local_md5sum($diskfiles[0]);
+            }
+            close(DD);
+            if ($good_file eq '') {
+                die "No valid instance of key: $key";
+            }
+            else {
+                print "cp $good_file $files[0]\n";
+                if ($do_ops) {
+                    system("cp $good_file $files[0]");
+                }
+            }
+            # Begin my best validation thought
+            {
+                my $tmpmd5 = local_md5sum($files[0]);
+                if ($tmpmd5 ne $md5sum) { 
+                    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+                }
+            }
+            # End my best validation thought.
+
+            $Ngood = 1;  # We now hand off this single valid instance object to be handled by the Ngood=1 case.
+            ## We've done work here, so we can't do a cull this iteration.
+            if ($do_cull == 1) { $do_cull = -1; }
+        }
+        ## We have more than zero bad copies.  We may cull some of these in the future, but we should try to
+        ## leave everything in the best state possible.
+        printf(">> %d %d\n",$Ngood, $#files + 1);
+        if ($Ngood != $#files + 1) {
+            my $good_copy;
+            my $good_copy_index;
+            foreach my $site_key (keys %good_instances) {
+                if ($#{ $good_instances{$site_key} } != -1) {
+                    $good_copy_index = $good_instances{$site_key}[0];
+                    $good_copy = $files[$good_copy_index];
+                    last;
+                }
+            }
+            printf(">> GOOD: $good_copy\n");
+            foreach my $site_key (keys %bad_instances) {
+                foreach my $bad_copy_index (@{ $bad_instances{$site_key} }) {
+                    print "cp $good_copy $files[$bad_copy_index]\n";
+                    if ($do_ops) {
+                        system("cp $good_copy $files[$bad_copy_index]");
+                    }
+                    my $tmpmd5 = local_md5sum($files[$bad_copy_index]);
+                    if ($tmpmd5 ne $md5sum) { 
+                    ## This isn't super critical, so we don't need to die here.
+                        warn "Post-repair md5sum does not match! $tmpmd5 != $md5sum: $files[$bad_copy_index]";
+                    }
+                    else {
+                    ## success
+                        push @{ $good_instances{$site_key} }, $bad_copy_index;
+                    }
+                }
+            }
+        ## We've done work here, so we can't do a cull this iteration.
+            if ($do_cull == 1) { $do_cull = -1; }
+        }
+
+            ## We can now attempt to make replicated copies to the sites that require additional copies.
+        foreach my $site_key (keys %requirement_map) {
+            my $have_instances = $#{ $good_instances{$site_key} } + 1;
+            print "## $site_key $have_instances $requirement_map{$site_key}\n";
+            if ($#{ $good_instances{$site_key} } + 1 < $requirement_map{$site_key}) {
+                my $rep_vol = get_random_site_volume($site_key);
+                print "neb-replicate --volume $rep_vol  $key\n";
+                if ($do_ops) {
+                    $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node";
+                    if ($@) { die $@; }
+                
+                # Begin my best validation thought
+                    system("sync") == 0 or die "Couldn't sync?";
+                    my $uris = $neb->find_instances($key,$rep_vol);
+                    @$uris = map {URI->new($_)->file if $_} @$uris;
+                    my $tmpmd5 = local_md5sum(${ $uris }[0]);
+
+                my $validation_str = sprintf("% 3d %d %32s %s %s %s",
+                                          -1,-1,$tmpmd5,
+                                             ${ $uris }[0],"repl",$rep_vol);
+                print
+                    join("\n" . " " x 4, $validation_str), "\n";
+                
                     if ($tmpmd5 ne $md5sum) { 
                         die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
                     }
-                }
-                # End my best validation thought.
-                
-                $Ngood = 1;  # We now hand off this single valid instance object to be handled by the Ngood=1 case.
-            }
-            
-            if ($quality[0] == 0) { # The first instance is bad.
-                # But since we're here, and not up there, there must be at least one good copy.  Find it.
-                for (my $i = 0; $i <= $#md5sums; $i++) {
-                    if ($md5sums[$i] eq $md5sum) { # Found it.
-                        print "cp $diskfiles[$i] $diskfiles[0]\n";
-                        vsystem("cp $diskfiles[$i] $diskfiles[0]");
-                        # Begin my best validation thought
-                        {
-                            my $tmpmd5 = local_md5sum($diskfiles[0]);
-                            if ($tmpmd5 ne $md5sum) { 
-                                die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-                            }
-                        }
-                        # End my best validation thought.
-                        last; # We're done here now.
-                    }
-                }       
-            }
-            
-            if ($Ngood == 1) { # We have only one version
-                if ($quality_mask & 1) { # ANd it's on a backup volume
-                    if ($user_copies > 1) { # And we want more than one copy.
-                        print "neb-replicate $key\n";
-                        if ($do_ops) {
-                            $neb->replicate($key) or die "failed to replicate the single valid copy";
-                            if ($@) { die $@; }
-                        }
-                    }
-                }
-                else { # And it's not, so put one there
-                    my $rep_vol = get_random_backup_volume();
-                    print "neb-replicate --volume $rep_vol  $key\n";
-                    if ($do_ops) {
-                        $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node";
-                        if ($@) { die $@; }
-                        
-                        # Begin my best validation thought
-                        vsystem("sync") == 0 or die "Couldn't sync?";
-                        my $uris = $neb->find_instances($key,$rep_vol);
-                        @$uris = map {URI->new($_)->file if $_} @$uris;
-                        my $tmpmd5 = local_md5sum(${ $uris }[0]);
-                        if ($tmpmd5 ne $md5sum) { 
-                            die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-                        }
-                        # End my best validation thought.
-                    }       
-                }
-            }
-            else { # N >= 2
-                if (!($quality_mask & 1)) { # no backup copy
-                    my $rep_vol = get_random_backup_volume();
-                    print "neb-replicate --volume $rep_vol  $key\n";
-                    if ($do_ops) {
-                        $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node";
-                        if ($@) { die $@; }
-                        
-                        # Begin my best validation thought
-                        vsystem("sync") == 0 or die "Couldn't sync?";
-                        my $uris = $neb->find_instances($key,$rep_vol);
-                        @$uris = map {URI->new($_)->file if $_} @$uris;
-                        my $tmpmd5 = local_md5sum(${ $uris }[0]);
-                        if ($tmpmd5 ne $md5sum) { 
-                            die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-                        }
-                        # End my best validation thought.
-                    }       
-                }
-                if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid.
-                    for (my $i = 0; $i <= $#diskfiles; $i++) {
-                        if ($quality[$i] == 2) {
-                            $quality[$i] = 1;
-                            last;
-                        }
-                    }
-                }
-            }
-#XXX NEW DEBUG THIS:  This should iterate over diskfiles that are not marked with a quality = 2, and cull them.  
-# For the N>=2, we set that for the primary, and
-# for both cases, we do not have a diskfile entry for the newly replicated copy.  
-# Therefore, this should cull down to the correct number.
-            for (my $i = 0; $i <= $#diskfiles; $i++) {
-#           print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n";
-                if ($quality[$i] != 1) {
-                    if ($existance[$i] == 0) { # This disk file doesn't exist.
-                        vsystem("touch $diskfiles[$i]");
-                    }
-                    my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
-                    print "neb-cull --volume $cull_vol $key\n";
-                    if ($do_ops) {
-                        # The tilde here is to force hard volumes.  Don't touch it.
-                        $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
-                        if ($@) { die "$@"; }
-                    }
-                }
-            }
-            # }
-        } # end keys for this product
-    } # end product for this entry
-} # end entry for this id
-
-sub vsystem {
-    my $cmd = shift;
-    print "$cmd\n";
-    if ($do_ops) { 
-        system($cmd);
-    }
-}
+                # End my best validation thought.
+                }       
+            ## We've done work here, so we can't do a cull this iteration.
+                if ($do_cull == 1) { $do_cull = -1; }
+            }
+        }
+    
+    ## Do culls if that's what we were going to do.
+        if ($do_cull == -1) {
+            die "Cull option passed, but files were modified in the scan/repair/replicate phase.  Not running cull!\n";
+        }
+        elsif ($do_cull == 1) {
+        ## At this point, we should have no files in the bad_instances lists, because we've repaired them.
+            foreach my $site_key (keys %good_instances) {
+                if ($#{ $good_instances{$site_key} } + 1 > $requirement_map{$site_key}) {
+                    for ($i = $requirement_map{$site_key}; $i <= $#{ $good_instances{$site_key} }; $i++) {
+                        my $cull_index = ${ $good_instances{$site_key} }[$i];
+                    my ($instance_host,$instance_volume) = parse_volume($files[$cull_index]);
+                    print "neb-cull --volume $instance_volume $key\n";
+                    if ($do_ops) {
+                        # The tilde here is to force hard volumes.  Don't touch it.
+                        # Also: the 2 is a "minimum number of copies" restriction.  Let's not be crazy here.
+                        $neb->cull($key,"~${instance_volume}",2) or die "failed to cull a superfluous instance";
+                        if ($@) { die "$@"; }
+                    }
+                } # End loop over extra instances
+            } # End check for sites with extra instances
+        } # End loop over sites.
+    } # End cull
+
+} ## End loop over imfiles.
+
 
 sub local_md5sum {
@@ -425 +527 @@
 }
 
-
-
 sub parse_volume {
     my $filename = shift(@_);
     my $full_volume   = (split /\//, $filename)[2];
-    my ($hostname,$vol_index) = split /\./, $full_volume; # /;
-    return($hostname,$vol_index);
-}
-
-sub is_backup_volume {
-    my $hostname = shift(@_);
-    if (exists($backup_hosts{$hostname})) {
-        return(1);
-    }
-    return(0);
-}
-
-sub get_random_backup_volume {
-    my $NN = scalar keys %backup_destinations;
-    my $backup_host = (keys %backup_destinations)[int(rand($NN))];
-    my $backup_vol  = int(rand($backup_Nvols));
-    
-    return("${backup_host}.${backup_vol}");
+    my ($hostname,undef) = split /\./, $full_volume; # /;
+    return($hostname,$full_volume);
+}
+
+sub get_random_site_volume {
+    my $site_key = shift(@_);
+    my $NN = scalar @{ $volume_lists{$site_key} };
+    my $backup_volume = ${ $volume_lists{$site_key} }[int(rand($NN))];
+    return($backup_volume);
 }