Changeset 42404

Timestamp:

Mar 9, 2023, 2:32:14 PM (3 years ago)

Author:

eugene

Message:

more updates to rawfix

Location:

trunk/tools/eam/rawfix.20230221/src

Files:

• check.rawfix.sh (modified) (2 diffs)
• check_md5s_dateobs.pl (added)
• fix_chip_locations.pl (modified) (1 diff)
• get_hosts_md5s.sh (modified) (1 diff)
• get_md5s_instances.pl (modified) (1 diff)
• gpc1.chips.txt (added)
• rawfix.pt (modified) (11 diffs)
• update.bnodes.sh (modified) (1 diff)

trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh

@@ -3 +3 @@
 if ($#argv < 1) then
   echo "USAGE: check.rawfix.sh (mode) [options]"
+  echo "USAGE: check.rawfix.sh all"
+  echo "USAGE: check.rawfix.sh stats"
   echo "USAGE: check.rawfix.sh night (YYYYMMDD)"
-  echo "USAGE: check.rawfix.sh new"
+  echo "USAGE: check.rawfix.sh state (state)"
+  echo " NOTE: state = raw, rsync, md5sum.new"
   exit 2
 endif
@@ -61 +64 @@
   if ($mystate == "rsync") goto good_state
   if ($mystate == "md5sum.new") goto good_state
+  if ($mystate == "md5sum.done") goto good_state
+  if ($mystate == "md5sum.fail") goto good_state
 
   echo "ERROR: unknown mode $mystate"

trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl

@@ -12 +12 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version );
 
-# USAGE: fix_chip_locations.pl --dateobs (night)
+# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file)
 
 # we need to select targets for ITC copies of files currently only at ATRC:

trunk/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh

@@ -14 +14 @@
 
 # report both the dateobs and the hostnames 
-ls $dirname/*.inst.txt | awk -v dateobs=$dirname -F/ '(NR == 1){print "dateobs md5host"}{print dateobs, $2}' | sed s/.inst.txt// | head 
+ls $dirname/*.inst.txt | awk -v dateobs=$dirname -F/ '(NR == 1){print "dateobs md5host"}{print dateobs, $2}' | sed s/.inst.txt//
 exit 0

trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl

@@ -27 +27 @@
 
 if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; }
+if (-e $output) {
+    # check that all instances are in the output file?
+    print "$output exists, skipping\n";
+    exit 0;
+}
+
 open (OUTFILE, ">$output");
 

trunk/tools/eam/rawfix.20230221/src/rawfix.pt

@@ -13 +13 @@
 
 # use this to find the files in this local directory
-$mypath = `pwd`
+$mypath = /home/panstarrs/eugene/rawfix.20230221
+
+if (not($?LOGDIR)) 
+  $LOGDIR = $mypath/pantasks_logs
+  mkdir $LOGDIR
+end
 
 if (not($?RAWFIX_INIT))
@@ -19 +24 @@
   book init md5host.book
   $RAWFIX_INIT = 1
-end
-
-macro load.hosts
-  # read the list of possible hosts from the file
-  # for each host, create a variable $host:name = 0
-  # when the host is running, set this to 1
-
-  list allhosts -x "cat allhosts.txt"
-
-  for i 0 $allhosts:n
-    $name = $allhosts:$i
-    $host:$name = 0
-  end
 end
 
@@ -42 +34 @@
   periods -timeout 30
 
+  stdout $LOGDIR/md5sum.load.stdout.log  
+  stderr $LOGDIR/md5sum.load.stderr.log
+
   # only run one of these tasks at a time
   npending 1
@@ -77 +72 @@
   periods      -timeout 60
   npending 1
+
+  stdout $LOGDIR/md5sum.loadhosts.stdout.log  
+  stderr $LOGDIR/md5sum.loadhosts.stderr.log
 
   task.exec
@@ -94 +92 @@
     option $pageName
     # save the page so we can use it below
+
+    # write an entry point in the log for this night
+    sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $DATEOBS
+    date -var mydate
+    exec echo "# start runhosts $mydate" >> $tmpline
 
     command get_hosts_md5s.sh $DATEOBS
@@ -133 +136 @@
   periods      -timeout 60
 
+  stdout $LOGDIR/md5sum.runhosts.stdout.log  
+  stderr $LOGDIR/md5sum.runhosts.stderr.log
+
   task.exec
     # if we are unable to run the 'exec', use a long retry time
@@ -152 +158 @@
     # save DATEOBS so we can find the md5sum.book entry
 
-    # XXX test: use test host
-    $TESTHOST = ipp060
-    host -required $TESTHOST
-    if (not($?host:$TESTHOST)) 
-      echo "host not defined, creating : $TESTHOST"
-      $host:$TESTHOST = 0
+    # XXX TEST: $USE_HOST = ipp060
+
+    $USE_HOST = $MD5HOST
+    host -required $USE_HOST
+    ## check if this host needs to be launched
+    if (not($?host:$USE_HOST)) 
+      echo "host not defined, creating : $USE_HOST"
+      $host:$USE_HOST = 0
     end
-    if (not($host:$TESTHOST)) 
-      control host add $TESTHOST
-      $host:$TESTHOST = 1
+    if (not($host:$USE_HOST)) 
+      control host add $USE_HOST
+      $host:$USE_HOST = 1
     end
-    # host -required $MD5HOST 
-
-    ## check if this host needs to be launched
-
+
+    ## even when testing, use the original hostname here:
     $input = $mypath/$DATEOBS/$MD5HOST.inst.txt
 
     # need to make sure this will work on the ipp b-nodes:
-    # command $mypath/get_md5s_instances.pl --hostname $MD5HOST --input $input
-    command $mypath/get_md5s_instances.pl --hostname $TESTHOST --input $input
+    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
+    periods -exec 0.05
+    command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input
   end
 
@@ -203 +210 @@
   periods      -timeout 60
   npending 1
+
+  stdout $LOGDIR/md5sum.clean.stdout.log  
+  stderr $LOGDIR/md5sum.clean.stderr.log
 
   task.exec
@@ -214 +224 @@
     # look for completed nights in md5sum.book (pantaskState == DONE)
     book getpage md5sum.book 0 -var pageName -key pantaskState DONE
-    echo "checking this page: $pageName"
     if ("$pageName" == "NULL") break
 
-    echo "cleaning this page: $pageName"
-    book listpage md5sum.book $pageName
+    # book listpage md5sum.book $pageName
 
     # book setword md5sum.book $pageName pantaskState RUN
     book getword md5sum.book $pageName dateobs -var DATEOBS
     book getword md5sum.book $pageName pantasksQuality -var QUALITY
-    echo "cleaning $pageName, quality: $QUALITY"
 
     option $pageName
@@ -229 +236 @@
 
     if ("$QUALITY" == "OK")
-       
       command update.rawfix.sh $DATEOBS md5sum.done 
     else
@@ -275 +281 @@
   if ("$1" != "OK")
     book setword md5sum.book $options:1 pantasksQuality $1
+    sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $options:1
+    exec echo "failure $1 $options:0" >> $tmpline
+  else
+    exec echo "success $1 $options:0" >> $tmpline
   end 
   if ($Ndone == $Nhosts)

trunk/tools/eam/rawfix.20230221/src/update.bnodes.sh

@@ -6 +6 @@
 endif
 
-set options = "-auvn"
+set options = "-auvn --exclude=trash"
 if ($2 == "commit") set options = "-auv"