Changeset 42415 for trunk/tools

Timestamp:

Mar 16, 2023, 12:48:31 PM (3 years ago)

Author:

eugene

Message:

various updates : adding stage 2

Location:

trunk/tools/eam/rawfix.20230221/src

Files:

• check.rawfix.sh (modified) (1 diff)
• check_chip_locations.pl (modified) (2 diffs)
• check_md5s_dateobs.pl (modified) (2 diffs)
• check_night_status.pl (added)
• fix_chip_locations.pl (modified) (5 diffs)
• fix_chip_locations_stage2.pl (added)
• get_md5s_instances_stage2.pl (added)
• rawfix.fixchip_s2.pt (added)
• rawfix.md5chk_s2.pt (added)
• rawfix.md5sum_s2.pt (added)
• rawfix.pt (modified) (2 diffs)

trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh

@@ -67 +67 @@
   if ($mystate == "fixchip.done")  goto good_state
   if ($mystate == "fixchip.fail")  goto good_state
-  if ($mystate == "rsync_s1.new")  goto good_state
-  if ($mystate == "rsync_s1.done") goto good_state
-  if ($mystate == "rsync_s1.fail") goto good_state
   if ($mystate == "md5sum.new")    goto good_state
   if ($mystate == "md5sum.done")   goto good_state
   if ($mystate == "md5sum.fail")   goto good_state
-  if ($mystate == "rsync_s2.new")  goto good_state
-  if ($mystate == "rsync_s2.done") goto good_state
-  if ($mystate == "rsync_s2.fail") goto good_state
   if ($mystate == "md5chk.new")    goto good_state
   if ($mystate == "md5chk.done")   goto good_state
   if ($mystate == "md5chk.fail")   goto good_state
+
+  # stage 2:
+  if ($mystate == "fixchip_s2.new")   goto good_state
+  if ($mystate == "fixchip_s2.done")  goto good_state
+  if ($mystate == "fixchip_s2.fail")  goto good_state
+  if ($mystate == "md5sum_s2.new")    goto good_state
+  if ($mystate == "md5sum_s2.done")   goto good_state
+  if ($mystate == "md5sum_s2.fail")   goto good_state
+  if ($mystate == "md5chk_s2.new")    goto good_state
+  if ($mystate == "md5chk_s2.done")   goto good_state
+  if ($mystate == "md5chk_s2.fail")   goto good_state
+
+# disable the rsyncs stages: run in directory from b-node 
+#  if ($mystate == "rsync_s1.new")  goto good_state
+#  if ($mystate == "rsync_s1.done") goto good_state
+#  if ($mystate == "rsync_s1.fail") goto good_state
+#  if ($mystate == "rsync_s2.new")  goto good_state
+#  if ($mystate == "rsync_s2.done") goto good_state
+#  if ($mystate == "rsync_s2.fail") goto good_state
 
   echo "ERROR: unknown mode $mystate"

trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl

@@ -67 +67 @@
         my ($Nmain, $Nback, $Ndead) = &check_instance_xattr ($inforows, $ext_id, $data_st, $md5_sum);
 
-        &check_instance_state ($outfh, $ext_id, $Nmain, $Nback, $Ndead);
+        &check_instance_state ($outfh, $ext_id, $data_st, $md5_sum, $Nmain, $Nback, $Ndead);
     }
 
@@ -249 +249 @@
     my $outfh = $_[0];
     my $extID = $_[1];
-    my $Nmain = $_[2];
-    my $Nback = $_[3];
-    my $Ndead = $_[4];
-
-    # SKIP : already fixed 
+    my $data_st = $_[2];
+    my $md5_sum = $_[3];
+    my $Nmain = $_[4];
+    my $Nback = $_[5];
+    my $Ndead = $_[6];
+
+    my $outinfo = "$extID $data_st $md5_sum";
+
+    # GOOD : already fixed 
     if (($Nmain == 1) && ($Nback == 1)) {
-        print $outfh "GOOD: $extID\n";
-        return;
-    }
-
-    # single copy @ ITC, no backup
+        print $outfh "GOOD: $outinfo\n";
+        return;
+    }
+
+    # SING : single copy @ ITC, no backup
     # make a backup
     if (($Nmain == 1) && ($Nback == 0)) {
-        print $outfh "SING: $extID\n";
-        return;
-    }
-
-    # ITC files with a single backup copies need to be culled
-    # cull Nmain - 1
-    if (($Nmain > 1) && ($Nback == 1)) {
-        print $outfh "CULL: $extID\n";
-        return;
-    }
-
-    # multiple copies @ ITC
+        print $outfh "SING: $outinfo\n";
+        return;
+    }
+
+    # SBCK : single copy @ ATRC, no main copy
+    # make a backup @ ITC
+    if (($Nmain == 0) && ($Nback == 1)) {
+        print $outfh "SBCK: $outinfo\n";
+        return;
+    }
+
+    # DUPS : multiple copies @ ITC, none at ATRC
     # make a backup and cull Nmain - 1
     if (($Nmain > 1) && ($Nback == 0)) {
-        print $outfh "DUPS: $extID\n";
-        return;
-    }
-
-    # no more valid instances
-    if (($Nmain == 0) && ($Nback == 0) && $Ndead) {
-        print $outfh "DEAD: $extID\n";
-        return;
-    }
-    if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) {
-        print $outfh "GONE: $extID\n";
-        return;
-    }
-
-    # single copy @ ATRC
-    # make a backup @ ITC
-    if (($Nmain == 0) && ($Nback == 1)) {
-        print $outfh "SBCK: $extID\n";
-        return;
-    }
-
-    # multiple copies @ ATRC, but none at ITC
+        print $outfh "DUPS: $outinfo\n";
+        return;
+    }
+
+    # XBCK : multiple copies @ ATRC, but none at ITC
     # make a backup at ITC, cull ATRC extras
     if (($Nmain == 0) &&  ($Nback > 1)) {
-        print $outfh "XBCK: $extID\n";
+        print $outfh "XBCK: $outinfo\n";
+        return;
+    }
+
+    # DEAD : no more valid instances
+    if (($Nmain == 0) && ($Nback == 0) && $Ndead) {
+        print $outfh "DEAD: $outinfo\n";
+        return;
+    }
+    # GONE : no more valid instances
+    if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) {
+        print $outfh "GONE: $outinfo\n";
+        return;
+    }
+
+    # CUL1 : multiple ITC files, single backup
+    # cull only ITC
+    if (($Nmain > 1) && ($Nback == 1)) {
+        print $outfh "CUL1: $outinfo\n";
+        return;
+    }
+
+    # CUL2 : multiple ATRC files, single ITC
+    # cull only ATRC
+    if (($Nback > 1) && ($Nmain == 1)) {
+        print $outfh "CUL2: $outinfo\n";
+        return;
+    }
+
+    # CUL3 : excess ATRC & ITC files, cull from both?
+    # cull Nmain - 1
+    if (($Nback > 1) && ($Nmain > 1)) {
+        print $outfh "CUL3: $outinfo\n";
         return;
     }
 
     # unexpected state:
-    print $outfh "OOPS: $extID\n";
+    print $outfh "OOPS: $outinfo\n";
     return;
 }

trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl

@@ -1 +1 @@
 #! /usr/bin/env perl
 
-my $DEBUG = 4;
+my $DEBUG = 1;
 my $time = time();
 
@@ -82 +82 @@
             print STDOUT "FAIL: $inst $state $md5tru $md5obs\n";
             $Nfail ++;
+        } else {
+            if ($DEBUG > 2) {
+                print STDOUT "$inst $state $md5tru = $md5obs\n";
+            }
         }
     }

trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl

@@ -36 +36 @@
 my $Nxbck = 0;
 
+my $Ncul1 = 0;
+my $Ncul2 = 0;
+my $Ncul3 = 0;
+
 foreach my $chip_id (@chip_ids) {
 
@@ -49 +53 @@
 
         # we expect lines of the form: STATE PATH
-        # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits
-        if (@words != 2) { die "error in URI list $urifile, line: $line\n"; }
+        # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum
+        if (@words != 4) { die "error in URI list $urifile, line: $line\n"; }
 
         my $state  = $words[0];
         my $ext_id = $words[1];
+        my $datast = $words[2];
+        my $md5sum = $words[3];
 
         if ($state eq "GOOD:") { $Ngood++; next; }
         if ($state eq "DEAD:") { $Ndead++; next; }
         if ($state eq "GONE:") { $Ngone++; next; }
-        if ($state eq "DUPS:") { $Ndups++; next; }
-        if ($state eq "XBCK:") { $Nxbck++; next; }
 
         # one copy at ITC: neb-replicate will create a copy at ATRC
@@ -68 +72 @@
             unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; }
             open (OUTFILE, ">>$dateword/$newhost.inst.txt");
-            print OUTFILE "$newfile\n";
+            print OUTFILE "$newfile $datast $md5sum\n";
             close (OUTFILE);
             $Nsing ++;
@@ -82 +86 @@
             unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; }
             open (OUTFILE, ">>$dateword/$newhost.inst.txt");
-            print OUTFILE "$newfile\n";
+            print OUTFILE "$newfile $datast $md5sum\n";
             close (OUTFILE);
             $Nsbck ++;
             next; 
         }
+
+        # these are done in stage 2:
+        if ($state eq "DUPS:") { $Ndups++; next; }
+        if ($state eq "XBCK:") { $Nxbck++; next; }
+
+        # these are done in stage 3:
+        if ($state eq "CUL1:") { $Ncul1++; next; }
+        if ($state eq "CUL2:") { $Ncul2++; next; }
+        if ($state eq "CUL3:") { $Ncul3++; next; }
     }
     $volnum ++;
@@ -98 +111 @@
 print "Nsbck: $Nsbck\n";
 print "Nxbck: $Nxbck\n";
+print "Ncul1: $Ncul1\n";
+print "Ncul2: $Ncul2\n";
+print "Ncul3: $Ncul3\n";
 
 exit 0;

trunk/tools/eam/rawfix.20230221/src/rawfix.pt

@@ -9 +9 @@
 
 # use this to find the files in this local directory
-$mypath = /home/panstarrs/eugene/rawfix.20230221
+# $mypath = /home/panstarrs/eugene/rawfix.20230221
+
+# NOTE: use a directory located on a b-node so the results can
+# be seen by both ATRC and ITC machines
+$mypath = /data/ippb07.0/home/eugene/rawfix.20230221
 
 if (not($?LOGDIR)) 
@@ -19 +23 @@
 # order is this file is not determinative, only the
 # change of state from STAGE1.new -> STAGE1.done -> STAGE2.new
-input rawfix.ckchip.pt
-input rawfix.fixchip.pt
-input rawfix.rsync_s1.pt
-input rawfix.md5sum.pt
-input rawfix.rsync_s2.pt
-input rawfix.md5chk.pt
+# input rawfix.ckchip.pt
+# input rawfix.fixchip.pt
+# input rawfix.md5sum.pt
+# input rawfix.md5chk.pt
+# input rawfix.fixchip_s2.pt
+input rawfix.md5sum_s2.pt
+#input rawfix.md5chk_s2.pt
+
+# no need to run rsync if we use the b-node working directory:
+# input rawfix.rsync_s1.pt
+# input rawfix.rsync_s2.pt
 
 ######################### UTILITIES #################################