Changeset 40178

Timestamp:

Oct 20, 2017, 4:14:55 PM (9 years ago)

Author:

watersc1

Message:

Updated shuffle code that fixes more cases including orphaned disk files (with no neb entry) and bad copies (exists but has wrong md5sum).

File:

• trunk/ippScripts/scripts/rawcheck_mddb.pl (modified) (11 diffs)

trunk/ippScripts/scripts/rawcheck_mddb.pl

@@ -208 +208 @@
         my $class_id   = $imfile->{class_id};
 
+        unless(defined($key))        { die "No database uri: $exp_id"; }
+        unless(defined($data_state)) { $data_state = 'no_data_state'; }
+        unless(defined($md5sum))     { die "No database md5sum: $exp_id"; }
+        unless(defined($hostname))   { $hostname = 'ipp004'; }
+        unless(defined($class_id))   { die "No database class_id: $class_id"; }
+
         # This hash contains all the information about the instances, indexed by the full filename of the instance.
         my %unified_list = ();
+        my %orphans      = ();
 
         $timer = time();
@@ -223 +230 @@
         for (my $i = 0; $i <= $#files; $i++) {
             my $f = $files[$i];
+            $f =~ s%//%/%g;
             my ($instance_exists,$instance_host,$instance_volume,$instance_site);
             ($instance_host,$instance_volume) = parse_volume($files[$i]);   
@@ -239 +247 @@
 
         # Get md database results for this exp_id/class_id
-
         my $md_query = "select file_id,ins_id,mvol_name,site_name,file,disk_sum,(STRCMP(md5sum,disk_sum) = 0) AS is_valid FROM obj JOIN ins USING(file_id) JOIN vol USING(mvol_id) WHERE exp_id = $exp_id AND class_id = '${class_id}'";
         my $md_instances = $mddb->selectall_arrayref( $md_query );
@@ -248 +255 @@
         foreach my $r (@{ $md_instances }) {
             my ($file_id,$ins_id,$vol_name,$site_name,$f,$disk_sum,$is_valid) = @{ $r };
-
+            $f =~ s%//%/%g;
             unless (exists($unified_list{$f})) {
                 if (-e $f) {
@@ -304 +311 @@
         # Check that evrything is consistent.
         my $good_copies = 0;
-        my $all_copies = length(keys(%unified_list));
+        my $all_copies = scalar(keys(%unified_list));
         my $a_good_copy = '';
 
@@ -315 +322 @@
             $unified_list{$f}{good_instance} = 0;
             print "$f  ";
-
+            
+            # Identify orphans, and push their handling until later.
             unless (exists($unified_list{$f}{neb_host})) {
-                die "No neb entry for $unified_list{$f}{md_ins_id} $exp_id $class_id $f \n";
-#               mddb_delete_instance($mddb,$unified_list{$f}{md_ins_id});
+                # Handle this.
+                $orphans{$f} = $unified_list{$f};
+                delete($unified_list{$f});
+                warn "No neb entry for $orphans{$f}{md_ins_id} $exp_id $class_id $f \n";
+                next;
             }
             print "$unified_list{$f}{neb_host} $unified_list{$f}{neb_vol} $unified_list{$f}{neb_site} $unified_list{$f}{exists}  ";
+
+
+            # Create an entry in the mddb database if that doesn't already exist.
             unless (exists($unified_list{$f}{md_vol})) {
                 # Insert a new row in the mddb ins table for this instance.  
+                if ($unified_list{$f}{exists} != 1) {
+                    # Ensure that the file exists, so an md5sum can be performed.
+                    system("touch $f");
+                }
                 my $tmpmd5 = local_md5sum($f);
                 my ($ins_id, $site_name) = mddb_insert_instance($mddb,$md_file_id,$f, $unified_list{$f}{neb_vol}, $tmpmd5);
+
+                $unified_list{$f}{md_vol}     = $unified_list{$f}{neb_vol};
+                $unified_list{$f}{md_site}    = $site_name;
+                $unified_list{$f}{disk_sum}   = $tmpmd5;
+                $unified_list{$f}{md_file_id} = $md_file_id;
+                $unified_list{$f}{md_ins_id}  = $ins_id;
 
                 if ($tmpmd5 ne $md5sum) { 
                     # Insert even if we fail so we can try fixing next time.
                     # This may cause a problem if the first instance we checked has a bad md5sum.
-                    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-                }
-
-                $unified_list{$f}{md_vol}     = $unified_list{$f}{neb_vol};
-                $unified_list{$f}{md_site}    = $site_name;
-                $unified_list{$f}{disk_sum}   = $tmpmd5;
-                $unified_list{$f}{is_valid}   = 1;
-                $unified_list{$f}{md_file_id} = $md_file_id;
-                $unified_list{$f}{md_ins_id}  = $ins_id;
+                    warn "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+                    $unified_list{$f}{is_valid}   = 0;
+                }
+                else {
+                    $unified_list{$f}{is_valid}   = 1;
+                }
             }
 
@@ -349 +370 @@
                 }
                 if ($tmpmd5 ne $md5sum) {
-                    die "Revalidated md5sum does not match!";
+                    warn "Revalidated md5sum does not match!";
                 }
             }
 
             print "$unified_list{$f}{md_file_id} $unified_list{$f}{md_ins_id} $unified_list{$f}{md_vol} $unified_list{$f}{md_site} $unified_list{$f}{disk_sum} $unified_list{$f}{is_valid}  ";
-            if ($unified_list{$f}{neb_vol} ne $unified_list{$f}{md_vol}) {
-                die "Inconsistent volumes for $exp_id $class_id $f $unified_list{$f}{neb_vol} $unified_list{$f}{md_vol}\n";
-            }
+
+
             if ($unified_list{$f}{exists} != 1) {
                 warn "Instance does not exist $exp_id $class_id $f\n";
                 next;
             }
+            if ($unified_list{$f}{neb_vol} ne $unified_list{$f}{md_vol}) {
+                die "Inconsistent volumes for $exp_id $class_id $f $unified_list{$f}{neb_vol} $unified_list{$f}{md_vol}\n";
+            }
             if ($unified_list{$f}{is_valid} != 1) {
                 warn "Instance is not valid $exp_id $class_id $f\n";
@@ -369 +392 @@
                 die "Inconsistent md5sum values $exp_id $class_id $f $md5sum $unified_list{$f}{disk_sum}\n";
             }
-
             
             # If none of that stopped us, then this instance is probably fine.
@@ -383 +405 @@
 
         if ($good_copies == 0) {
-            die "No good copies available for $exp_id $class_id\n";
+            my $orphan_copy = '';
+            foreach my $orph (keys (%orphans)) {
+                if ($orphans{$orph}{is_valid} == 1) {
+                    $orphan_copy = $orph;
+                }
+            }
+
+            if ($orphan_copy eq '') {
+                die "No good copies available for $exp_id $class_id\n";
+            }
+            else {
+                die "No good copies available for $exp_id $class_id (but $orphan_copy may exist)\n";
+            }
         }
 
@@ -390 +424 @@
 
         # Attempt to repair any bad instance.
+        print "COUNT CHECK $good_copies $all_copies\n";
         if ($good_copies != $all_copies) {
             foreach my $f (keys(%unified_list)) {
@@ -497 +532 @@
                 }
             }
+
+            # If we're here, then we've sucessfully sorted out this instance.
+            foreach my $f (keys %orphans) {
+                print "ORPHAN $f $orphans{$f}{md_ins_id}\n";
+                mddb_delete_instance($mddb,$orphans{$f}{md_ins_id});
+#               unlink($f) or warn "Could not unlink $f: $!";
+            }
+            
         } # End culling