Changeset 42452

Timestamp:

Apr 21, 2023, 10:36:27 AM (3 years ago)

Author:

eugene

Message:

some improvements (make sure to examine md5s for chip-specific inst files, randomly draw the new nights)

Location:

trunk/tools/eam/rawfix.20230221/src

Files:

• advance.rawfix.pl (modified) (1 diff)
• check_chip_locations.pl (modified) (1 diff)
• check_md5s_dateobs.pl (modified) (5 diffs)
• get_hosts_md5s.sh (modified) (1 diff)
• rawfix.advance.pt (modified) (3 diffs)
• rawfix.fixchip.pt (modified) (3 diffs)
• rawfix.md5sum.pt (modified) (3 diffs)

trunk/tools/eam/rawfix.20230221/src/advance.rawfix.pl

@@ -77 +77 @@
 
     # select and advance nights not yet running
-    $sql_get_run = "SELECT dateobs from nights where state = 'new'";
+#   $sql_get_run = "SELECT dateobs from nights where state = 'new'";
+
+    $sql_get_run = "SELECT dateobs FROM (SELECT dateobs, rand() AS myrand FROM nights WHERE (state = 'new')) AS tmp ORDER BY myrand";
+
     $runref   = $raw_dbh->selectall_arrayref( $sql_get_run );
     @runarray = @$runref;

trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl

@@ -196 +196 @@
         if ($Nvalues != 3) { print "ERROR: invalid response?\n"; next; }
         
+        # data_state and possibly md5sum can be NULL, need to handle specially
+        my $dstate = $values[1]; if (not defined $dstate) { $dstate = "NULL"; } 
+        my $md5sum = $values[2]; if (not defined $md5sum) { $md5sum = "XXXX"; } 
+        
         push (@nebarray, $values[0]);
-        push (@ds_array, $values[1]);
-        push (@md_array, $values[2]);
-        if ($DEBUG > 4) { print "$values[0]\n $values[1]\n $values[2]\n"; }
+        push (@md_array, $md5sum);
+        push (@ds_array, $dstate);
+
+        if ($DEBUG > 5) { print "V0: $values[0]\n"; print "V1: $values[1]\n"; print "V2: $values[2]\n"; }
+        if ($DEBUG > 4) { print "PRINT: $values[0]\n $dstate\n $md5sum\n"; }
     }
 

trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl

@@ -16 +16 @@
 my ($dateword, $stage, $topdir) = &parse_cmdopts;
 
-my @instlist = glob ("$topdir/$dateword/*.inst.$stage.txt");
-my @md5slist = glob ("$topdir/$dateword/*.md5s.txt");
+my $instlist_r = &load_instlist_filenames ($topdir, $dateword, $stage);
+my $md5slist_r = &load_md5slist_filenames ($topdir, $dateword, $stage, $instlist_r);
 
 my $Nfail = 0;
@@ -23 +23 @@
 my %goodchips;
 
-foreach my $instfile (@instlist) {
-
-    # find the matching md5sum file for this host
-
-    my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|;
-    # print STDERR "host: $host\n";
-
-    my $md5match = "NONE";
-    foreach my $md5sfile (@md5slist) {
-        
-        my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|;
-        # print STDERR "md5host: $md5shost\n";
-        
-        if ($host ne $md5shost) { next; }
-
-        $md5match = $md5sfile;
-        last;
-    }
-
-    if ($md5match eq "NONE") {
-        print STDOUT "missing md5sum file for $instfile\n";
-        $Nfail ++;
-        next;
-    }
+# make a list of the master instance files:
+
+foreach my $instfile (@$instlist_r) {
+
+    # print "inst: $instfile\n";
+
+    my $md5sfile = $md5slist_r->{$instfile};
 
     ## read in the md5sums
-    open (FILE, $md5match);
+    open (FILE, $md5sfile);
     my @md5data = <FILE>;
     close (FILE);
 
-    ## read in the list of instances
-    open (FILE, $instfile);
-    my @instdata = <FILE>;
-    close (FILE);
+    my $instdata_r = &load_instance_list ($topdir, $dateword, $stage, $instfile);
 
     # if these do not match, complain and move on
@@ -62 +42 @@
     # may have been culled during this process (but the MD5s stay behind)
     my $N1 = @md5data;
-    my $N2 = @instdata;
+    my $N2 = @$instdata_r;
     if ($N1 < $N2) {
-        print STDOUT "not enough MD5s $md5match : $N1 vs $N2\n";
+        print STDOUT "not enough MD5s $md5sfile : $N1 vs $N2\n";
         $Nfail ++;
         next;
+    } else {
+        # print STDOUT "checking MD5s $md5sfile : $N1 vs $N2\n";
     }
 
     # set up an empty hash for the instance names:
     my %instfound = ();
-    foreach my $line (@instdata) {
+    foreach my $line (@$instdata_r) {
         chomp $line;
         my @words = split (/\s+/,$line);
@@ -86 +68 @@
 
         # we expect lines of the form: INST STATE MDtru MDobs
-        if (@words != 4) { die "error in MD5 list $md5match, line: $line\n"; }
+        if (@words != 4) { die "error in MD5 list $md5sfile, line: $line\n"; }
 
         my $inst   = $words[0];
@@ -136 +118 @@
     print STDOUT "SUCCESS: $Nfail errors\n";
     exit 0;
+}
+
+# use global $topdir, $dateword, $myhost, $stage
+sub load_instance_list {
+    my $topdir   = $_[0];
+    my $dateword = $_[1];
+    my $stage    = $_[2];
+    my $instfile = $_[3];
+
+    my @instdata_list;
+
+    # load the master list
+    &load_instance_list_onefile ($instfile, \@instdata_list);
+
+    # there may be additional instances in files with names like HOSTNAME.XYnn.inst.STAGE 
+    my ($myhost) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|;
+    my @instlist = glob ("$topdir/$dateword/$myhost.XY??.inst.$stage.txt");
+    foreach my $infile  (@instlist) {
+        &load_instance_list_onefile ($infile, \@instdata_list);
+    }
+    return (\@instdata_list);
+}
+
+# read the instance info
+sub load_instance_list_onefile {
+    my $infile = $_[0];
+    my $instdata_list_r = $_[1]; # reference to the file data
+
+    open(FILE,$infile) or die "cannot open file $infile\n"; 
+    my @lines = <FILE>;
+    close (FILE);
+
+    foreach my $line (@lines) {
+        chomp $line;
+        if ($line =~ /^\s*#/) { next; } # skip lines with leading #
+        push (@$instdata_list_r, $line);
+    }
+}
+
+# return only the master instlist filenames (ignore ones with the chipnames in the name)
+sub load_instlist_filenames {
+    
+    my $topdir   = $_[0];
+    my $dateword = $_[1];
+    my $stage    = $_[2];
+
+    my @instlist_all = glob ("$topdir/$dateword/*.inst.$stage.txt");
+
+    my @instlist = ();
+
+    foreach my $instfile (@instlist_all) {
+
+        # skip the per-chip inst files (these should probably be merged into the master)
+        if ($instfile =~ m|XY[0-9][0-9].inst|) { next; }
+
+        push (@instlist, $instfile);
+    }
+    return (\@instlist);
+}
+
+# return a hash of the matched md5 file for each inst file
+sub load_md5slist_filenames {
+
+    my $topdir     = $_[0];
+    my $dateword   = $_[1];
+    my $stage      = $_[2];
+    my $instlist_r = $_[3];
+
+    my @md5slist = glob ("$topdir/$dateword/*.md5s.txt");
+
+    my %md5shash;
+
+    my $Nfail = 0;
+
+    foreach my $instfile (@$instlist_r) {
+
+        # find the matching md5sum file for this host
+
+        my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|;
+        # print STDERR "host: $host\n";
+
+        my $md5match = "NONE";
+        foreach my $md5sfile (@md5slist) {
+        
+            my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|;
+        
+            if ($host ne $md5shost) { next; }
+
+            # print STDERR "md5host: $md5shost : $md5sfile = $instfile\n";
+
+            $md5match = $md5sfile;
+            last;
+        }
+
+        if ($md5match eq "NONE") {
+            print STDOUT "missing md5sum file for $instfile\n";
+            $Nfail ++;
+            next;
+        }
+        $md5shash{$instfile} = $md5match;
+
+        # print "match: $instfile = $md5match\n";
+
+    }
+    if ($Nfail > 0) { 
+        print STDOUT "ERROR: $Nfail errors\n";
+        exit 1;
+    }
+    return (\%md5shash);
 }
 

trunk/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh

@@ -21 +21 @@
 
 # report both the dateobs and the hostnames 
-ls $topdir/$dateobs/*.inst.$stage.txt | awk -v dateobs=$dateobs -v stage=$stage -F/ '(NR == 1){print "dateobs stage md5host"}{print dateobs, stage, $NF}' | sed s/.inst.$stage.txt//
+ls $topdir/$dateobs/*.inst.$stage.txt | grep -v "XY...inst" | awk -v dateobs=$dateobs -v stage=$stage -F/ '(NR == 1){print "dateobs stage md5host"}{print dateobs, stage, $NF}' | sed s/.inst.$stage.txt//
 exit 0

trunk/tools/eam/rawfix.20230221/src/rawfix.advance.pt

@@ -1 +1 @@
 
-$MAX_ACTIVE_NIGHTS = 5
+$MAX_ACTIVE_NIGHTS = 10
 
 ## regularly run the command advance.rawfits.sh
@@ -8 +8 @@
   periods -timeout 30
   host local
+  active true
 
   stdout $LOGDIR/advance.log  
@@ -39 +40 @@
   end
 end
+
+macro nights.advance.on
+  task nights.advance
+    active true
+  end
+end
+
+macro nights.advance.off
+  task nights.advance
+    active false
+  end
+end
+
+  

trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

@@ -12 +12 @@
   $RAWFIX_FIXCHIP_INIT = 1
 end
+
+$MAX_FIXHOST = 2
 
 # select nights in state fixchip.new
@@ -97 +99 @@
     # convert the stdout 'queue' to entries in a book
     # fields in the table will become words in the page
-    queueprint stdout
+    # queueprint stdout
     queuesize stdout -var Nhosts
     queue2book -raw-columns -key dateobs:chipID -uniq -setword pantaskState INIT stdout fixhost.book
     book setword fixchip.book $options:0 NHOST {$Nhosts - 1}
     book setword fixchip.book $options:0 NDONE 0
-    book list
-    book listbook fixchip.book
+    # book list
+    # book listbook fixchip.book
     # use the queuesize (Nhosts) to track how many jobs are done / to be done
   end
@@ -163 +165 @@
       $host:$FIXHOST = 0
     end
-    if (not($host:$FIXHOST)) 
+    if ($host:$FIXHOST < $MAX_FIXHOST) 
       control host add $FIXHOST
-      $host:$FIXHOST = 1
+      $host:$FIXHOST ++
     end
 

trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt

@@ -12 +12 @@
   $RAWFIX_MD5SUM_INIT = 1
 end
+
+$MAX_MD5HOST = 2
 
 # select nights which are ready for md5sum calcs
@@ -97 +99 @@
     # convert the stdout 'queue' to entries in a book
     # fields in the table will become words in the page
-    queueprint stdout
+    # queueprint stdout
     queuesize stdout -var Nhosts
     queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host.book
     book setword md5sum.book $options:0 NHOST {$Nhosts - 1}
     book setword md5sum.book $options:0 NDONE 0
-    book list
-    book listbook md5sum.book
+    # book list
+    # book listbook md5sum.book
     # use the queuesize (Nhosts) to track how many jobs are done / to be done
   end
@@ -166 +168 @@
       $host:$USE_HOST = 0
     end
-    if (not($host:$USE_HOST)) 
+    if ($host:$USE_HOST < $MAX_MD5HOST) 
       control host add $USE_HOST
-      $host:$USE_HOST = 1
+      $host:$USE_HOST ++
     end