Changeset 42452 for trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl

Timestamp:

Apr 21, 2023, 10:36:27 AM (3 years ago)

Author:

eugene

Message:

some improvements (make sure to examine md5s for chip-specific inst files, randomly draw the new nights)

File:

• trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl (modified) (5 diffs)

trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl

@@ -16 +16 @@
 my ($dateword, $stage, $topdir) = &parse_cmdopts;
 
-my @instlist = glob ("$topdir/$dateword/*.inst.$stage.txt");
-my @md5slist = glob ("$topdir/$dateword/*.md5s.txt");
+my $instlist_r = &load_instlist_filenames ($topdir, $dateword, $stage);
+my $md5slist_r = &load_md5slist_filenames ($topdir, $dateword, $stage, $instlist_r);
 
 my $Nfail = 0;
@@ -23 +23 @@
 my %goodchips;
 
-foreach my $instfile (@instlist) {
-
-    # find the matching md5sum file for this host
-
-    my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|;
-    # print STDERR "host: $host\n";
-
-    my $md5match = "NONE";
-    foreach my $md5sfile (@md5slist) {
-        
-        my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|;
-        # print STDERR "md5host: $md5shost\n";
-        
-        if ($host ne $md5shost) { next; }
-
-        $md5match = $md5sfile;
-        last;
-    }
-
-    if ($md5match eq "NONE") {
-        print STDOUT "missing md5sum file for $instfile\n";
-        $Nfail ++;
-        next;
-    }
+# make a list of the master instance files:
+
+foreach my $instfile (@$instlist_r) {
+
+    # print "inst: $instfile\n";
+
+    my $md5sfile = $md5slist_r->{$instfile};
 
     ## read in the md5sums
-    open (FILE, $md5match);
+    open (FILE, $md5sfile);
     my @md5data = <FILE>;
     close (FILE);
 
-    ## read in the list of instances
-    open (FILE, $instfile);
-    my @instdata = <FILE>;
-    close (FILE);
+    my $instdata_r = &load_instance_list ($topdir, $dateword, $stage, $instfile);
 
     # if these do not match, complain and move on
@@ -62 +42 @@
     # may have been culled during this process (but the MD5s stay behind)
     my $N1 = @md5data;
-    my $N2 = @instdata;
+    my $N2 = @$instdata_r;
     if ($N1 < $N2) {
-        print STDOUT "not enough MD5s $md5match : $N1 vs $N2\n";
+        print STDOUT "not enough MD5s $md5sfile : $N1 vs $N2\n";
         $Nfail ++;
         next;
+    } else {
+        # print STDOUT "checking MD5s $md5sfile : $N1 vs $N2\n";
     }
 
     # set up an empty hash for the instance names:
     my %instfound = ();
-    foreach my $line (@instdata) {
+    foreach my $line (@$instdata_r) {
         chomp $line;
         my @words = split (/\s+/,$line);
@@ -86 +68 @@
 
         # we expect lines of the form: INST STATE MDtru MDobs
-        if (@words != 4) { die "error in MD5 list $md5match, line: $line\n"; }
+        if (@words != 4) { die "error in MD5 list $md5sfile, line: $line\n"; }
 
         my $inst   = $words[0];
@@ -136 +118 @@
     print STDOUT "SUCCESS: $Nfail errors\n";
     exit 0;
+}
+
+# use global $topdir, $dateword, $myhost, $stage
+sub load_instance_list {
+    my $topdir   = $_[0];
+    my $dateword = $_[1];
+    my $stage    = $_[2];
+    my $instfile = $_[3];
+
+    my @instdata_list;
+
+    # load the master list
+    &load_instance_list_onefile ($instfile, \@instdata_list);
+
+    # there may be additional instances in files with names like HOSTNAME.XYnn.inst.STAGE 
+    my ($myhost) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|;
+    my @instlist = glob ("$topdir/$dateword/$myhost.XY??.inst.$stage.txt");
+    foreach my $infile  (@instlist) {
+        &load_instance_list_onefile ($infile, \@instdata_list);
+    }
+    return (\@instdata_list);
+}
+
+# read the instance info
+sub load_instance_list_onefile {
+    my $infile = $_[0];
+    my $instdata_list_r = $_[1]; # reference to the file data
+
+    open(FILE,$infile) or die "cannot open file $infile\n"; 
+    my @lines = <FILE>;
+    close (FILE);
+
+    foreach my $line (@lines) {
+        chomp $line;
+        if ($line =~ /^\s*#/) { next; } # skip lines with leading #
+        push (@$instdata_list_r, $line);
+    }
+}
+
+# return only the master instlist filenames (ignore ones with the chipnames in the name)
+sub load_instlist_filenames {
+    
+    my $topdir   = $_[0];
+    my $dateword = $_[1];
+    my $stage    = $_[2];
+
+    my @instlist_all = glob ("$topdir/$dateword/*.inst.$stage.txt");
+
+    my @instlist = ();
+
+    foreach my $instfile (@instlist_all) {
+
+        # skip the per-chip inst files (these should probably be merged into the master)
+        if ($instfile =~ m|XY[0-9][0-9].inst|) { next; }
+
+        push (@instlist, $instfile);
+    }
+    return (\@instlist);
+}
+
+# return a hash of the matched md5 file for each inst file
+sub load_md5slist_filenames {
+
+    my $topdir     = $_[0];
+    my $dateword   = $_[1];
+    my $stage      = $_[2];
+    my $instlist_r = $_[3];
+
+    my @md5slist = glob ("$topdir/$dateword/*.md5s.txt");
+
+    my %md5shash;
+
+    my $Nfail = 0;
+
+    foreach my $instfile (@$instlist_r) {
+
+        # find the matching md5sum file for this host
+
+        my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|;
+        # print STDERR "host: $host\n";
+
+        my $md5match = "NONE";
+        foreach my $md5sfile (@md5slist) {
+        
+            my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|;
+        
+            if ($host ne $md5shost) { next; }
+
+            # print STDERR "md5host: $md5shost : $md5sfile = $instfile\n";
+
+            $md5match = $md5sfile;
+            last;
+        }
+
+        if ($md5match eq "NONE") {
+            print STDOUT "missing md5sum file for $instfile\n";
+            $Nfail ++;
+            next;
+        }
+        $md5shash{$instfile} = $md5match;
+
+        # print "match: $instfile = $md5match\n";
+
+    }
+    if ($Nfail > 0) { 
+        print STDOUT "ERROR: $Nfail errors\n";
+        exit 1;
+    }
+    return (\%md5shash);
 }