Changeset 42411 for trunk/tools

Timestamp:

Mar 14, 2023, 3:08:37 PM (3 years ago)

Author:

eugene

Message:

more work on the rawfix scripts

Location:

trunk/tools/eam/rawfix.20230221/src

Files:

• check.rawfix.sh (modified) (2 diffs)
• check_md5s_dateobs.pl (modified) (4 diffs)
• get_md5s_instances.pl (modified) (1 diff)
• rawfix.ckchip.pt (modified) (2 diffs)
• rawfix.fixchip.pt (modified) (2 diffs)
• rawfix.md5chk.pt (added)
• rawfix.md5sum.pt (modified) (4 diffs)
• rawfix.pt (modified) (1 diff)
• rawfix.rsync_s1.pt (added)
• rawfix.rsync_s2.pt (added)

trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh

@@ -61 +61 @@
   set mystate = $2
 
-  if ($mystate == "new") goto good_state
-  if ($mystate == "rsync") goto good_state
-  if ($mystate == "md5sum.new") goto good_state
-  if ($mystate == "md5sum.done") goto good_state
-  if ($mystate == "md5sum.fail") goto good_state
-  if ($mystate == "ckchip.new") goto good_state
-  if ($mystate == "ckchip.done") goto good_state
-  if ($mystate == "ckchip.fail") goto good_state
-  if ($mystate == "fixchip.new") goto good_state
-  if ($mystate == "fixchip.done") goto good_state
-  if ($mystate == "fixchip.fail") goto good_state
+  if ($mystate == "ckchip.new")    goto good_state
+  if ($mystate == "ckchip.done")   goto good_state
+  if ($mystate == "ckchip.fail")   goto good_state
+  if ($mystate == "fixchip.new")   goto good_state
+  if ($mystate == "fixchip.done")  goto good_state
+  if ($mystate == "fixchip.fail")  goto good_state
+  if ($mystate == "rsync_s1.new")  goto good_state
+  if ($mystate == "rsync_s1.done") goto good_state
+  if ($mystate == "rsync_s1.fail") goto good_state
+  if ($mystate == "md5sum.new")    goto good_state
+  if ($mystate == "md5sum.done")   goto good_state
+  if ($mystate == "md5sum.fail")   goto good_state
+  if ($mystate == "rsync_s2.new")  goto good_state
+  if ($mystate == "rsync_s2.done") goto good_state
+  if ($mystate == "rsync_s2.fail") goto good_state
+  if ($mystate == "md5chk.new")    goto good_state
+  if ($mystate == "md5chk.done")   goto good_state
+  if ($mystate == "md5chk.fail")   goto good_state
 
   echo "ERROR: unknown mode $mystate"
@@ -77 +84 @@
 
   good_state:
-  mysql $DBOPTS -e "select dateobs from nights where (state = '$mystate')"
+  set t = `date +%T..%N`
+  mysql $DBOPTS -e "select dateobs, curtime() as timestamp from nights where (state = '$mystate')" | awk -v T=$t '(NR == 1){print $0}(NR > 1){print $1, T}'
   exit 0
 endif

trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl

@@ -18 +18 @@
 my @instlist = glob ("$dateword/*.inst.txt");
 my @md5slist = glob ("$dateword/*.md5s.txt");
+
+my $Nfail = 0;
 
 foreach my $instfile (@instlist) {
@@ -39 +41 @@
 
     if ($md5match eq "NONE") {
-        print STDERR "missing md5sum file for $instfile\n";
+        print STDOUT "missing md5sum file for $instfile\n";
+        $Nfail ++;
         next;
     }
@@ -57 +60 @@
     my $N2 = @instdata;
     if ($N1 != $N2) {
-        print STDERR "inconsistent number of lines in $md5match : $N1 vs $N2\n";
+        print STDOUT "inconsistent number of lines in $md5match : $N1 vs $N2\n";
+        $Nfail ++;
         next;
     }
@@ -76 +80 @@
 
         if ($md5tru ne $md5obs) {
-            print STDERR "FAIL: $inst $state $md5tru $md5obs\n";
+            print STDOUT "FAIL: $inst $state $md5tru $md5obs\n";
+            $Nfail ++;
         }
     }
+}
+
+if ($Nfail) { 
+    print STDOUT "ERROR: $Nfail errors\n";
+    exit 1;
+} else {
+    print STDOUT "SUCCESS: $Nfail errors\n";
+    exit 0;
 }
 

trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl

@@ -4 +4 @@
 my $time = time();
 
-use DBI;
 use Carp;
 use Pod::Usage qw( pod2usage );

trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt

@@ -30 +30 @@
   end
 
+  # need to delete DONE entries here so the book is not repopulated
+  # by already-running load tasks
   task.exit 0
     # convert the stdout 'queue' to entries in a book
     # fields in the table will become words in the page
     queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout ckchip.book
+    book delpage ckchip.book -key pantaskState DONE
   end
 
@@ -91 +94 @@
     # fields in the table will become words in the page
     # use the queuesize (Nhosts) to track how many jobs are done / to be done
+    book setword ckchip.book $options:0 pantaskState DONE
     exec update.rawfix.sh $options:0 ckchip.done
-    book delpage ckchip.book $options:0
   end
 

trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

@@ -35 +35 @@
     # fields in the table will become words in the page
     queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip.book
+    book delpage fixchip.book -key pantaskState DONE
   end
 
@@ -91 +92 @@
     # fields in the table will become words in the page
     # use the queuesize (Nhosts) to track how many jobs are done / to be done
+    book setword fixchip.book $options:0 pantaskState DONE
     exec update.rawfix.sh $options:0 fixchip.done
     book delpage fixchip.book $options:0

trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt

@@ -35 +35 @@
     # fields in the table will become words in the page
     queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum.book
+    book delpage md5sum.book -key pantaskState DONE
   end
 
@@ -210 +211 @@
     
     # look for completed nights in md5sum.book (pantaskState == DONE)
-    book getpage md5sum.book 0 -var pageName -key pantaskState DONE
+    book getpage md5sum.book 0 -var pageName -key pantaskState STOP
     if ("$pageName" == "NULL") break
 
@@ -232 +233 @@
     # now that the entry is set to done in the db, remove the pantasks book entry
     echo "cleaning this page: $options:0"
-    book delpage md5sum.book $options:0
+    book setword md5sum.book $options:0 pantaskState DONE
   end
 
@@ -274 +275 @@
   end 
   if ($Ndone == $Nhosts)
-    book setword md5sum.book $options:1 pantaskState DONE
+    book setword md5sum.book $options:1 pantaskState STOP
   end
   book delpage md5host.book $options:0

trunk/tools/eam/rawfix.20230221/src/rawfix.pt

@@ -16 +16 @@
 end
 
+# these are listed in stage order, but the 
+# order is this file is not determinative, only the
+# change of state from STAGE1.new -> STAGE1.done -> STAGE2.new
 input rawfix.ckchip.pt
-# input rawfix.md5sum.pt
+input rawfix.fixchip.pt
+input rawfix.rsync_s1.pt
+input rawfix.md5sum.pt
+input rawfix.rsync_s2.pt
+input rawfix.md5chk.pt
 
 ######################### UTILITIES #################################