Changeset 42531 for branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt

Timestamp:

Aug 17, 2023, 4:41:34 PM (3 years ago)

Author:

eugene

Message:

merging from trunk

Location:

branches/eam_branches/ipp-20230313/tools

Files:

• . (modified) (1 prop)
• eam/rawfix.20230221/src/rawfix.md5sum.pt (modified) (13 diffs)

branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt

@@ -12 +12 @@
   $RAWFIX_MD5SUM_INIT = 1
 end
+
+$MAX_MD5HOST = 2
 
 # select nights which are ready for md5sum calcs
@@ -35 +37 @@
     # fields in the table will become words in the page
     queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum.book
+    book delpage md5sum.book -key pantaskState DONE
   end
 
@@ -54 +57 @@
 # output: md5host.book
 task nights.md5sum.loadhosts
-
   periods      -poll $RPOLL
   periods      -exec $REXEC
   periods      -timeout 60
+  host          local
   npending 1
+  # note: the tasks which return their results via stdout
+  # seem to fail if run remotely.  
 
   stdout $LOGDIR/md5sum.loadhosts.stdout.log  
@@ -77 +82 @@
     book setword md5sum.book $pageName pantaskState RUN
     book getword md5sum.book $pageName dateobs -var DATEOBS
+    book getword md5sum.book $pageName iter    -var ITER
     option $pageName
     # save the page so we can use it below
 
+    substr $DATEOBS 0 4 YEAR
+
     # write an entry point in the log for this night
-    sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $DATEOBS
+    sprintf tmpline "%s/%s/%s/log.md5sum.runhosts.v%d" $mypath $YEAR $DATEOBS $ITER
     date -var mydate
     exec echo "# start runhosts $mydate" >> $tmpline
 
-    command get_hosts_md5s.sh $DATEOBS
+    # the command returns these fields: dateobs stage md5host
+    # NOTE: stage == ITER
+    command get_hosts_md5s.sh $mypath $DATEOBS $ITER
   end
 
@@ -91 +101 @@
     # convert the stdout 'queue' to entries in a book
     # fields in the table will become words in the page
+    # queueprint stdout
     queuesize stdout -var Nhosts
     queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host.book
     book setword md5sum.book $options:0 NHOST {$Nhosts - 1}
     book setword md5sum.book $options:0 NDONE 0
+    # book list
+    # book listbook md5sum.book
     # use the queuesize (Nhosts) to track how many jobs are done / to be done
   end
@@ -116 +129 @@
 end
 
+# runs on the hosts identified by get_hosts_md5s.sh
 # output: md5host.book
-task nights.md5sum.runhosts
+task nights.md5sum.run
 
   periods      -poll $RPOLL
   periods      -exec $REXEC
   periods      -timeout 60
-
-  stdout $LOGDIR/md5sum.runhosts.stdout.log  
-  stderr $LOGDIR/md5sum.runhosts.stderr.log
 
   task.exec
@@ -138 +149 @@
     if ("$pageName" == "NULL") break
 
+    # XXX NOTE: the output field is called 'stage'
     book setword md5host.book $pageName pantaskState RUN
     book getword md5host.book $pageName dateobs -var DATEOBS
+    book getword md5host.book $pageName stage   -var ITER
     book getword md5host.book $pageName md5host -var MD5HOST
 
-    option $pageName $DATEOBS
+    substr $DATEOBS 0 4 YEAR
+
+    option $pageName $DATEOBS $YEAR
     # save DATEOBS so we can find the md5sum.book entry
-
-    # XXX TEST: $USE_HOST = ipp060
+    # also save YEAR for output log
+
+    # does the output go here?
+    stdout $mypath/$YEAR/$DATEOBS/log.md5sum.$MD5HOST.v$ITER  
+    stderr $mypath/$YEAR/$DATEOBS/log.md5sum.$MD5HOST.v$ITER  
 
     $USE_HOST = $MD5HOST
@@ -154 +172 @@
       $host:$USE_HOST = 0
     end
-    if (not($host:$USE_HOST)) 
+    if ($host:$USE_HOST < $MAX_MD5HOST) 
       control host add $USE_HOST
-      $host:$USE_HOST = 1
+      $host:$USE_HOST ++
     end
 
     ## even when testing, use the original hostname here:
-    $input = $mypath/$DATEOBS/$MD5HOST.inst.txt
+    $input = $mypath/$DATEOBS/$MD5HOST.inst.$ITER.txt
 
     # need to make sure this will work on the ipp b-nodes:
     # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
     periods -exec 0.05
-    command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input
+    command get_md5s_instances.pl --hostname $USE_HOST --topdir $mypath --dateobs $DATEOBS --stage $ITER
   end
 
   task.exit 0
-    nights.mdsum.exit OK
+    nights.md5sum.exit OK
   end
 
   # exit values other than 0:
   task.exit    default
-    nights.mdsum.exit FAIL
+    nights.md5sum.exit FAIL
     showcommand failure
   end
 
   task.exit    crash
-    nights.mdsum.exit CRASH
+    nights.md5sum.exit CRASH
     showcommand crash
   end
@@ -185 +203 @@
   # operation times out?
   task.exit    timeout
-    nights.mdsum.exit TIMEOUT
+    nights.md5sum.exit TIMEOUT
     showcommand timeout
   end
@@ -210 +228 @@
     
     # look for completed nights in md5sum.book (pantaskState == DONE)
-    book getpage md5sum.book 0 -var pageName -key pantaskState DONE
+    book getpage md5sum.book 0 -var pageName -key pantaskState STOP
     if ("$pageName" == "NULL") break
 
@@ -232 +250 @@
     # now that the entry is set to done in the db, remove the pantasks book entry
     echo "cleaning this page: $options:0"
-    book delpage md5sum.book $options:0
+    book setword md5sum.book $options:0 pantaskState DONE
   end
 
@@ -255 +273 @@
 ######################### UTILITIES #################################
 
-macro nights.mdsum.exit
+macro nights.md5sum.exit
   if ($0 != 2)
-    echo "USAGE: nights.mdsum.exit (quality)"
+    echo "USAGE: nights.md5sum.exit (quality)"
     break
   end
+
+  # options are : pagename DATEOBS YEAR
 
   book setword md5host.book $options:0 pantaskState DONE
   book getword md5sum.book $options:1 NHOST -var Nhosts
   book getword md5sum.book $options:1 NDONE -var Ndone
+  book getword md5sum.book $options:1 stage -var ITER
   $Ndone ++
   book setword md5sum.book $options:1 NDONE $Ndone
+
+  sprintf tmpline "%s/%s/%s/log.md5sum.runhosts.v%d" $mypath $options:2 $options:1 $ITER
   if ("$1" != "OK")
     book setword md5sum.book $options:1 pantasksQuality $1
-    sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $options:1
     exec echo "failure $1 $options:0" >> $tmpline
   else
@@ -274 +296 @@
   end 
   if ($Ndone == $Nhosts)
-    book setword md5sum.book $options:1 pantaskState DONE
+    book setword md5sum.book $options:1 pantaskState STOP
   end
   book delpage md5host.book $options:0