Changeset 39986

Timestamp:

Feb 24, 2017, 5:36:47 PM (9 years ago)

Author:

watersc1

Message:

Updated rawcheck.pl that attempts to check if it can skip any components if they've already been copied. The logic being that if components 1,2,3 have an ITC copy, but 4 doesn't, the error occurred on component 3, so only 3+ need to be checked. The prescan constructs a new array containing only these entries. This can be a problem with the md5sum checking, so all ITC copies get full md5sum calculations during the regular scan, so the initial copy of component 3 gets an md5sum (to find a valid copy), and whatever ITC copy of component 3 also gets an md5sum (to confirm the existing file is valid). Starting with component 4 this will switch back to checking only the initial copy (as there isn't a pre-existing ITC copy).

Tested on some previous failed exp_ids, and confirmed that the prescan is skipped entirely for ISP data (which have just the one component anyway).

File:

• trunk/ippScripts/scripts/rawcheck.pl (modified) (3 diffs)

trunk/ippScripts/scripts/rawcheck.pl

@@ -175 +175 @@
 my $mdcParser = PS::IPP::Metadata::Config->new;
 
-print("$regtool -processedimfile -exp_id $exp_id -dbname $dbname");
+print("$regtool -processedimfile -exp_id $exp_id -dbname $dbname\n");
 my $regtool_cmd = "$regtool -processedimfile -exp_id $exp_id -dbname $dbname";
 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
@@ -189 +189 @@
 my $timer = time();
 print "## rawcheck.pl: $exp_id $dbname $do_cull $do_ops $timer_start\n";
+# Do prescan checks.
+if ($#{ $imfiles } > 0) {
+    my @to_do_imfiles = ();
+    my $prior_imfile;
+    my $append = 0;
+
+    foreach my $imfile (@$imfiles) {
+        my $key        = $imfile->{uri};
+        my $data_state = $imfile->{data_state};
+        my $md5sum     = $imfile->{md5sum};
+        my $hostname   = $imfile->{hostname};
+        my $class_id   = $imfile->{class_id};
+        
+        my $stat = $neb->stat($key);
+        die "nebulous key: $key not found" unless $stat;   
+        my $instances = $neb->find_instances($key, 'any');
+        die "no instances found" unless $instances;   
+        my @files = map {URI->new($_)->file if $_} @$instances;
+        
+        print ("\n#PRE: $key $data_state $md5sum $hostname $class_id T: $timer\n");
+        
+        my $has_itc_copy = 0;
+        my @validation = ();
+        for (my $i = 0; $i <= $#files; $i++) {
+            # Check for itc copy.
+            my ($instance_exists,$instance_md5sum,$instance_host,$instance_volume,$instance_site, $is_good);
+            ($instance_host,$instance_volume) = parse_volume($files[$i]);
+            $instance_site = $volume_map{$instance_volume};
+            
+            if ($instance_site eq 'ITC') {
+                $has_itc_copy = 1;
+            }
+            
+            
+            $validation[$i] = sprintf("    % 3d %d %32s %s %s %s %s\n",
+                                      -1,-1,"UNCHECKED",
+                                      $files[$i],$instance_host,$instance_volume,$instance_site);
+            
+        }
+        my $val_string = join('',@validation);
+        print "$val_string";
+        
+        if ($append == 0) {
+            if ($has_itc_copy == 0) { 
+                if (defined($prior_imfile)) { 
+                    push @to_do_imfiles, $prior_imfile;
+                }
+                $append = 1; 
+            }
+        }
+        
+        if ($append == 1) {
+            push @to_do_imfiles, $imfile;
+        }
+        printf( "#PRE_RESULT: %2d %2d %2d %2d\n",$has_itc_copy,$append,$#to_do_imfiles,$#{ $imfiles });
+        $prior_imfile = $imfile;
+    }
+    
+    @$imfiles = @to_do_imfiles;     
+}
+# End prescan checks.
+
 # Loop over the imfiles of this exposure.
 foreach my $imfile (@$imfiles) {
@@ -230 +292 @@
             $instance_exists = 1;
             $existing_copies++;
+            if ($instance_site eq 'ITC') {  # This is here to force checking of ITC copy.
+                $continue_checking_md5sums = 1;
+            }
             if ($continue_checking_md5sums == 1) { #unsafe
                 $instance_md5sum = local_md5sum($files[$i]);