Changeset 42404 for trunk/tools/eam/rawfix.20230221/src/rawfix.pt

Timestamp:

Mar 9, 2023, 2:32:14 PM (3 years ago)

Author:

eugene

Message:

more updates to rawfix

File:

• trunk/tools/eam/rawfix.20230221/src/rawfix.pt (modified) (11 diffs)

trunk/tools/eam/rawfix.20230221/src/rawfix.pt

@@ -13 +13 @@
 
 # use this to find the files in this local directory
-$mypath = `pwd`
+$mypath = /home/panstarrs/eugene/rawfix.20230221
+
+if (not($?LOGDIR)) 
+  $LOGDIR = $mypath/pantasks_logs
+  mkdir $LOGDIR
+end
 
 if (not($?RAWFIX_INIT))
@@ -19 +24 @@
   book init md5host.book
   $RAWFIX_INIT = 1
-end
-
-macro load.hosts
-  # read the list of possible hosts from the file
-  # for each host, create a variable $host:name = 0
-  # when the host is running, set this to 1
-
-  list allhosts -x "cat allhosts.txt"
-
-  for i 0 $allhosts:n
-    $name = $allhosts:$i
-    $host:$name = 0
-  end
 end
 
@@ -42 +34 @@
   periods -timeout 30
 
+  stdout $LOGDIR/md5sum.load.stdout.log  
+  stderr $LOGDIR/md5sum.load.stderr.log
+
   # only run one of these tasks at a time
   npending 1
@@ -77 +72 @@
   periods      -timeout 60
   npending 1
+
+  stdout $LOGDIR/md5sum.loadhosts.stdout.log  
+  stderr $LOGDIR/md5sum.loadhosts.stderr.log
 
   task.exec
@@ -94 +92 @@
     option $pageName
     # save the page so we can use it below
+
+    # write an entry point in the log for this night
+    sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $DATEOBS
+    date -var mydate
+    exec echo "# start runhosts $mydate" >> $tmpline
 
     command get_hosts_md5s.sh $DATEOBS
@@ -133 +136 @@
   periods      -timeout 60
 
+  stdout $LOGDIR/md5sum.runhosts.stdout.log  
+  stderr $LOGDIR/md5sum.runhosts.stderr.log
+
   task.exec
     # if we are unable to run the 'exec', use a long retry time
@@ -152 +158 @@
     # save DATEOBS so we can find the md5sum.book entry
 
-    # XXX test: use test host
-    $TESTHOST = ipp060
-    host -required $TESTHOST
-    if (not($?host:$TESTHOST)) 
-      echo "host not defined, creating : $TESTHOST"
-      $host:$TESTHOST = 0
+    # XXX TEST: $USE_HOST = ipp060
+
+    $USE_HOST = $MD5HOST
+    host -required $USE_HOST
+    ## check if this host needs to be launched
+    if (not($?host:$USE_HOST)) 
+      echo "host not defined, creating : $USE_HOST"
+      $host:$USE_HOST = 0
     end
-    if (not($host:$TESTHOST)) 
-      control host add $TESTHOST
-      $host:$TESTHOST = 1
+    if (not($host:$USE_HOST)) 
+      control host add $USE_HOST
+      $host:$USE_HOST = 1
     end
-    # host -required $MD5HOST 
-
-    ## check if this host needs to be launched
-
+
+    ## even when testing, use the original hostname here:
     $input = $mypath/$DATEOBS/$MD5HOST.inst.txt
 
     # need to make sure this will work on the ipp b-nodes:
-    # command $mypath/get_md5s_instances.pl --hostname $MD5HOST --input $input
-    command $mypath/get_md5s_instances.pl --hostname $TESTHOST --input $input
+    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
+    periods -exec 0.05
+    command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input
   end
 
@@ -203 +210 @@
   periods      -timeout 60
   npending 1
+
+  stdout $LOGDIR/md5sum.clean.stdout.log  
+  stderr $LOGDIR/md5sum.clean.stderr.log
 
   task.exec
@@ -214 +224 @@
     # look for completed nights in md5sum.book (pantaskState == DONE)
     book getpage md5sum.book 0 -var pageName -key pantaskState DONE
-    echo "checking this page: $pageName"
     if ("$pageName" == "NULL") break
 
-    echo "cleaning this page: $pageName"
-    book listpage md5sum.book $pageName
+    # book listpage md5sum.book $pageName
 
     # book setword md5sum.book $pageName pantaskState RUN
     book getword md5sum.book $pageName dateobs -var DATEOBS
     book getword md5sum.book $pageName pantasksQuality -var QUALITY
-    echo "cleaning $pageName, quality: $QUALITY"
 
     option $pageName
@@ -229 +236 @@
 
     if ("$QUALITY" == "OK")
-       
       command update.rawfix.sh $DATEOBS md5sum.done 
     else
@@ -275 +281 @@
   if ("$1" != "OK")
     book setword md5sum.book $options:1 pantasksQuality $1
+    sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $options:1
+    exec echo "failure $1 $options:0" >> $tmpline
+  else
+    exec echo "success $1 $options:0" >> $tmpline
   end 
   if ($Ndone == $Nhosts)