Changeset 42449 for trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

Timestamp:

Apr 13, 2023, 9:30:40 AM (3 years ago)

Author:

eugene

Message:

split fix_chip_locations into parallel processes by chip

File:

• trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt (modified) (8 diffs)

trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

@@ -9 +9 @@
 if (not($?RAWFIX_FIXCHIP_INIT))
   book init fixchip.book
+  book init fixhost.book
   $RAWFIX_FIXCHIP_INIT = 1
 end
@@ -27 +28 @@
 
   task.exec
-    # command check.rawfix.sh state ckchip.done
     command check.rawfix.sh state fixchip.new 
   end
@@ -53 +53 @@
 end
 
-task nights.fixchip.run
+# output: fixhost.book
+task nights.fixchip.loadhosts
   periods      -poll $RPOLL
   periods      -exec $REXEC
-  periods      -timeout 1800
-  host         -required $HOST_FIXCHIP
+  periods      -timeout 60
+  host          local
   npending 1
+  # note: the tasks which return their results via stdout
+  # seem to fail if run remotely.  
+
+  stdout $LOGDIR/fixchip.loadhosts.stdout.log  
+  stderr $LOGDIR/fixchip.loadhosts.stderr.log
 
   task.exec
@@ -76 +82 @@
     book getword fixchip.book $pageName iter    -var ITER
     option $pageName
-
-    stdout $pageName/log.fixchip.v$ITER
-    stderr $pageName/log.fixchip.v$ITER
+    # save the page so we can use it below
+
+    # write an entry point in the log for this night
+    sprintf tmpline "%s/%s/log.fixchip.runhosts.v%d" $mypath $DATEOBS $ITER
+    date -var mydate
+    exec echo "# start runhosts $mydate" >> $tmpline
+
+    # the command returns these fields: dateobs stage fixhost chipID
+    # NOTE: stage == ITER
+    command $mypath/get_hosts_fixchip.sh $mypath $DATEOBS $ITER
+  end
+
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    queueprint stdout
+    queuesize stdout -var Nhosts
+    queue2book -raw-columns -key dateobs:chipID -uniq -setword pantaskState INIT stdout fixhost.book
+    book setword fixchip.book $options:0 NHOST {$Nhosts - 1}
+    book setword fixchip.book $options:0 NDONE 0
+    book list
+    book listbook fixchip.book
+    # use the queuesize (Nhosts) to track how many jobs are done / to be done
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+    book setword fixchip.book $options:0 pantasksQuality FAIL
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword fixchip.book $options:0 pantasksQuality CRASH
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword fixchip.book $options:0 pantasksQuality TIMEOUT
+  end
+end
+
+task nights.fixchip.run
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 1800
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages fixhost.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new nights in fixhost.book (pantaskState == INIT)
+    book getpage fixhost.book 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword fixhost.book $pageName pantaskState RUN
+    book getword fixhost.book $pageName dateobs -var DATEOBS
+    book getword fixhost.book $pageName iter    -var ITER
+    book getword fixhost.book $pageName fixhost -var FIXHOST
+    book getword fixhost.book $pageName chipID  -var CHIP_ID
+    option $pageName $DATEOBS
+
+    stdout $pageName/log.fixchip.run.v$ITER
+    stderr $pageName/log.fixchip.run.v$ITER
 
     substr $DATEOBS 0 4 YEAR
@@ -85 +157 @@
     sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
 
-    command $mypath/fix_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage $ITER
-  end
-
-  ## XX delete the page from the book if done?
-  task.exit 0
+    host -required $FIXHOST
+    ## check if this host needs to be launched
+    if (not($?host:$FIXHOST)) 
+      echo "host not defined, creating : $FIXHOST"
+      $host:$FIXHOST = 0
+    end
+    if (not($host:$FIXHOST)) 
+      control host add $FIXHOST
+      $host:$FIXHOST = 1
+    end
+
+    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
+    periods -exec 0.05
+    command $mypath/fix_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --chipID $CHIP_ID --stage $ITER
+  end
+
+  task.exit 0
+    nights.fixchip.exit OK
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    nights.fixchip.exit FAIL
+    showcommand failure
+  end
+
+  task.exit    crash
+    nights.fixchip.exit CRASH
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    nights.fixchip.exit TIMEOUT
+    showcommand timeout
+  end
+end
+
+# output: fixhost.book
+task nights.fixchip.clean
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+  npending 1
+
+  stdout $LOGDIR/fixchip.clean.stdout.log  
+  stderr $LOGDIR/fixchip.clean.stderr.log
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages fixchip.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for completed nights in fixchip.book (pantaskState == STOP)
+    book getpage fixchip.book 0 -var pageName -key pantaskState STOP
+    if ("$pageName" == "NULL") break
+
+    # book listpage fixchip.book $pageName
+
+    # book setword fixchip.book $pageName pantaskState RUN
+    book getword fixchip.book $pageName dateobs -var DATEOBS
+    book getword fixchip.book $pageName pantasksQuality -var QUALITY
+
+    option $pageName
+    # save the page so we can use it below
+
+    if ("$QUALITY" == "OK")
+      command update.rawfix.sh $DATEOBS fixchip.done 
+    else
+      command update.rawfix.sh $DATEOBS fixchip.fail 
+    end
+  end
+
+  task.exit 0
+    # now that the entry is set to done in the db, remove the pantasks book entry
+    echo "cleaning this page: $options:0"
     book setword fixchip.book $options:0 pantaskState DONE
-    exec update.rawfix.sh $options:0 fixchip.done
   end
 
@@ -98 +244 @@
     showcommand failure
     book setword fixchip.book $options:0 pantasksQuality FAIL
-    exec update.rawfix.sh $options:0 fixchip.fail
   end
 
@@ -104 +249 @@
     showcommand crash
     book setword fixchip.book $options:0 pantasksQuality CRASH
-    exec update.rawfix.sh $options:0 fixchip.fail
   end
 
@@ -111 +255 @@
     showcommand timeout
     book setword fixchip.book $options:0 pantasksQuality TIMEOUT
-    exec update.rawfix.sh $options:0 fixchip.fail
-  end
-end
-
+  end
+end
+
+######################### UTILITIES #################################
+
+macro nights.fixchip.exit
+  if ($0 != 2)
+    echo "USAGE: nights.fixchip.exit (quality)"
+    break
+  end
+
+  book setword fixhost.book $options:0 pantaskState DONE
+  book getword fixchip.book $options:1 NHOST -var Nhosts
+  book getword fixchip.book $options:1 NDONE -var Ndone
+  book getword fixchip.book $options:1 stage -var ITER
+  $Ndone ++
+  book setword fixchip.book $options:1 NDONE $Ndone
+  if ("$1" != "OK")
+    book setword fixchip.book $options:1 pantasksQuality $1
+    sprintf tmpline "%s/%s/log.fixchip.runhosts.v%d" $mypath $options:1 $ITER
+    exec echo "failure $1 $options:0" >> $tmpline
+  else
+    exec echo "success $1 $options:0" >> $tmpline
+  end 
+  if ($Ndone == $Nhosts)
+    book setword fixchip.book $options:1 pantaskState STOP
+  end
+  book delpage fixhost.book $options:0
+end
+