Changeset 42447 for trunk/tools

Timestamp:

Apr 11, 2023, 11:00:58 AM (3 years ago)

Author:

eugene

Message:

fix little issues

Location:

trunk/tools/eam/rawfix.20230221/src

Files:

• check_md5s_dateobs.pl (modified) (3 diffs)
• fix_chip_locations.pl (modified) (2 diffs)
• rawfix.ckchip.pt (modified) (2 diffs)
• rawfix.fixchip.pt (modified) (1 diff)
• rawfix.md5chk.pt (modified) (2 diffs)
• rawfix.pt (modified) (2 diffs)

trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl

@@ -59 +59 @@
 
     # if these do not match, complain and move on
+    # we are allowed to have more MD5s than instances: some instances
+    # may have been culled during this process (but the MD5s stay behind)
     my $N1 = @md5data;
     my $N2 = @instdata;
-    if ($N1 != $N2) {
-        print STDOUT "inconsistent number of lines in $md5match : $N1 vs $N2\n";
+    if ($N1 < $N2) {
+        print STDOUT "not enough MD5s $md5match : $N1 vs $N2\n";
         $Nfail ++;
         next;
     }
 
+    # set up an empty hash for the instance names:
+    my %instfound = ();
+    foreach my $line (@instdata) {
+        chomp $line;
+        my @words = split (/\s+/,$line);
+        my $inst   = $words[0];
+        $instfound{$inst} = 0;
+    }
+
     # check for mismatched MD5sums
+    # we are not checking that all instances in the inst file are in the MD5 file
     foreach my $line (@md5data) {
 
@@ -80 +92 @@
         my $md5tru = $words[2];
         my $md5obs = $words[3];
+
+        unless (defined $instfound{$inst}) { print "skip md5 for removed instance $inst\n"; next; }
+        $instfound{$inst} = 1;
 
         if ($md5tru ne $md5obs) {
@@ -88 +103 @@
             if ($DEBUG > 2) { print STDOUT "$inst $state $md5tru = $md5obs\n"; }
         }
+    }
+
+    my @instfoundKEYS = keys %instfound;
+    foreach my $inst (@instfoundKEYS) {
+        unless ($instfound{$inst}) { die "missing MD5 for instance $inst\n"; }
     }
 }

trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl

@@ -195 +195 @@
                 my $NcullTotal = @$cull_list_r;
                 foreach my $instance (@$cull_list_r) {
+                    print "state $state : checking instance $instance ... ";
                     my ($volname) = $instance =~ m|file:///data/([0-9a-zA-Z]*\.[0-9])/\S*|;
+                    print "volname $volname\n";
                     unless (defined $vol_good{$volname}) { die "ERROR: invalid volume name $volname\n"; }
                     # print STDERR "neb-cull $state: $ext_id $volname ($main_keep $back_keep)\n";
@@ -321 +323 @@
         if (($xattr == 1) && ($back_keep eq "")) { $back_keep = $instance; next; }
 
+        if ($xattr > 1) { print "ignore lost instance $instance\n"; next; }
+
         # if we make it here, we already have an instance at this location
         push (@cull_list, $instance);

trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt

@@ -58 +58 @@
   periods      -exec $REXEC
   periods      -timeout 60
-  host          anyhost
+  host         -required $HOST_CKCHIP
   npending 1
 
@@ -89 +89 @@
     sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
 
+    # this command accesses the Nebulous and GPC databases, so it must run at ITC.
     command $mypath/check_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage $ITER
   end

trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

@@ -57 +57 @@
   periods      -exec $REXEC
   periods      -timeout 1800
-  host          anyhost
+  host         -required $HOST_FIXCHIP
   npending 1
 

trunk/tools/eam/rawfix.20230221/src/rawfix.md5chk.pt

@@ -53 +53 @@
 
 task nights.md5chk.run
-  host anyhost
   periods      -poll $RPOLL
   periods      -exec $REXEC
   periods      -timeout 60
+  host          anyhost
   npending 1
 
@@ -85 +85 @@
     stderr $mypath/$DATEOBS/log.md5chk.v$ITER
 
+    ## this command does NOT use Nebulous or GPC so it can run anywhere
     command $mypath/check_md5s_dateobs.pl --topdir $mypath --dateobs $FULLDATE --stage $ITER
   end

trunk/tools/eam/rawfix.20230221/src/rawfix.pt

@@ -15 +15 @@
 $mypath = /data/ippb07.0/home/eugene/rawfix.20230221
 
+# I need to avoid ATRC machines for jobs which try to access Nebulous or GPC
+# I cannot define an avoidance list for a job, so I need to specify the 
+# required host
+
+$HOST_CKCHIP  = ipp060
+$HOST_FIXCHIP = ipp070
+
 if (not($?LOGDIR)) 
   $LOGDIR = $mypath/pantasks_logs
@@ -28 +35 @@
 input rawfix.md5sum.pt
 input rawfix.md5chk.pt
+
+if (not($?INIT_RAWFIX)) 
+  control host add $HOST_CKCHIP
+  control host add $HOST_FIXCHIP
+  $INIT_RAWFIX = 1
+end
 
 ######################### UTILITIES #################################