Changeset 42531 for branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

Timestamp:

Aug 17, 2023, 4:41:34 PM (3 years ago)

Author:

eugene

Message:

merging from trunk

Location:

branches/eam_branches/ipp-20230313/tools

Files:

• . (modified) (1 prop)
• eam/rawfix.20230221/src/rawfix.fixchip.pt (modified) (8 diffs)

branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

@@ -9 +9 @@
 if (not($?RAWFIX_FIXCHIP_INIT))
   book init fixchip.book
+  book init fixhost.book
   $RAWFIX_FIXCHIP_INIT = 1
 end
+
+$MAX_FIXHOST = 2
 
 # select nights in state fixchip.new
@@ -27 +30 @@
 
   task.exec
-    # command check.rawfix.sh state ckchip.done
     command check.rawfix.sh state fixchip.new 
   end
@@ -35 +37 @@
     # fields in the table will become words in the page
     queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip.book
-  end
-
-  # exit values other than 0:
-  task.exit    default
-    showcommand failure
-  end
-
-  task.exit    crash
-    showcommand crash
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-
-task nights.fixchip.run
-
+    book delpage fixchip.book -key pantaskState DONE
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# output: fixhost.book
+task nights.fixchip.loadhosts
   periods      -poll $RPOLL
   periods      -exec $REXEC
   periods      -timeout 60
+  host          local
   npending 1
-
-  stdout $LOGDIR/fixchip.run.stdout.log  
-  stderr $LOGDIR/fixchip.run.stderr.log
+  # note: the tasks which return their results via stdout
+  # seem to fail if run remotely.  
+
+  stdout $LOGDIR/fixchip.loadhosts.stdout.log  
+  stderr $LOGDIR/fixchip.loadhosts.stderr.log
 
   task.exec
@@ -76 +82 @@
     book setword fixchip.book $pageName pantaskState RUN
     book getword fixchip.book $pageName dateobs -var DATEOBS
+    book getword fixchip.book $pageName iter    -var ITER
     option $pageName
+    # save the page so we can use it below
+
+    substr $DATEOBS 0 4 YEAR
+
+    # write an entry point in the log for this night
+    sprintf tmpline "%s/%s/%s/log.fixchip.runhosts.v%d" $mypath $YEAR $DATEOBS $ITER
+    date -var mydate
+    exec echo "# start runhosts $mydate" >> $tmpline
+
+    # the command returns these fields: dateobs stage fixhost chipID
+    # NOTE: stage == ITER
+    command get_hosts_fixchip.sh $mypath $DATEOBS $ITER
+  end
+
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    # queueprint stdout
+    queuesize stdout -var Nhosts
+    queue2book -raw-columns -key dateobs:chipID -uniq -setword pantaskState INIT stdout fixhost.book
+    book setword fixchip.book $options:0 NHOST {$Nhosts - 1}
+    book setword fixchip.book $options:0 NDONE 0
+    # book list
+    # book listbook fixchip.book
+    # use the queuesize (Nhosts) to track how many jobs are done / to be done
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+    book setword fixchip.book $options:0 pantasksQuality FAIL
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword fixchip.book $options:0 pantasksQuality CRASH
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword fixchip.book $options:0 pantasksQuality TIMEOUT
+  end
+end
+
+task nights.fixchip.run
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 1800
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages fixhost.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new nights in fixhost.book (pantaskState == INIT)
+    book getpage fixhost.book 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword fixhost.book $pageName pantaskState RUN
+    book getword fixhost.book $pageName dateobs -var DATEOBS
+    book getword fixhost.book $pageName iter    -var ITER
+    book getword fixhost.book $pageName fixhost -var FIXHOST
+    book getword fixhost.book $pageName chipID  -var CHIP_ID
 
     substr $DATEOBS 0 4 YEAR
@@ -83 +157 @@
     sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
 
-    command fix_chip_locations.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt
-  end
-
-  ## XX delete the page from the book if done?
-  task.exit 0
-    # convert the stdout 'queue' to entries in a book
-    # fields in the table will become words in the page
-    # use the queuesize (Nhosts) to track how many jobs are done / to be done
-    exec update.rawfix.sh $options:0 fixchip.done
-    book delpage fixchip.book $options:0
+    option $pageName $DATEOBS $YEAR
+    # save DATEOBS so we can find the md5sum.book entry
+    # also save YEAR for output log
+
+    stdout $mypath/$YEAR/$DATEOBS/log.fixchip.run.$CHIP_ID.v$ITER
+    stderr $mypath/$YEAR/$DATEOBS/log.fixchip.run.$CHIP_ID.v$ITER
+
+    host -required $FIXHOST
+    ## check if this host needs to be launched
+    if (not($?host:$FIXHOST)) 
+      echo "host not defined, creating : $FIXHOST"
+      $host:$FIXHOST = 0
+    end
+    if ($host:$FIXHOST < $MAX_FIXHOST) 
+      control host add $FIXHOST
+      $host:$FIXHOST ++
+    end
+
+    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
+    periods -exec 0.05
+    command fix_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --chipID $CHIP_ID --stage $ITER
+  end
+
+  task.exit 0
+    nights.fixchip.exit OK
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    nights.fixchip.exit FAIL
+    showcommand failure
+  end
+
+  task.exit    crash
+    nights.fixchip.exit CRASH
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    nights.fixchip.exit TIMEOUT
+    showcommand timeout
+  end
+end
+
+# output: fixhost.book
+task nights.fixchip.clean
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+  npending 1
+
+  stdout $LOGDIR/fixchip.clean.stdout.log  
+  stderr $LOGDIR/fixchip.clean.stderr.log
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages fixchip.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for completed nights in fixchip.book (pantaskState == STOP)
+    book getpage fixchip.book 0 -var pageName -key pantaskState STOP
+    if ("$pageName" == "NULL") break
+
+    # book listpage fixchip.book $pageName
+
+    # book setword fixchip.book $pageName pantaskState RUN
+    book getword fixchip.book $pageName dateobs -var DATEOBS
+    book getword fixchip.book $pageName pantasksQuality -var QUALITY
+
+    option $pageName
+    # save the page so we can use it below
+
+    if ("$QUALITY" == "OK")
+      command update.rawfix.sh $DATEOBS fixchip.done 
+    else
+      command update.rawfix.sh $DATEOBS fixchip.fail 
+    end
+  end
+
+  task.exit 0
+    # now that the entry is set to done in the db, remove the pantasks book entry
+    echo "cleaning this page: $options:0"
+    book setword fixchip.book $options:0 pantaskState DONE
   end
 
@@ -99 +251 @@
     showcommand failure
     book setword fixchip.book $options:0 pantasksQuality FAIL
-    exec update.rawfix.sh $options:0 fixchip.fail
   end
 
@@ -105 +256 @@
     showcommand crash
     book setword fixchip.book $options:0 pantasksQuality CRASH
-    exec update.rawfix.sh $options:0 fixchip.fail
   end
 
@@ -112 +262 @@
     showcommand timeout
     book setword fixchip.book $options:0 pantasksQuality TIMEOUT
-    exec update.rawfix.sh $options:0 fixchip.fail
-  end
-end
-
+  end
+end
+
+######################### UTILITIES #################################
+
+macro nights.fixchip.exit
+  if ($0 != 2)
+    echo "USAGE: nights.fixchip.exit (quality)"
+    break
+  end
+
+  # options are : pagename DATEOBS YEAR
+
+  book setword fixhost.book $options:0 pantaskState DONE
+  book getword fixchip.book $options:1 NHOST -var Nhosts
+  book getword fixchip.book $options:1 NDONE -var Ndone
+  book getword fixchip.book $options:1 stage -var ITER
+  $Ndone ++
+  book setword fixchip.book $options:1 NDONE $Ndone
+
+  sprintf tmpline "%s/%s/%s/log.fixchip.runhosts.v%d" $mypath $options:2 $options:1 $ITER
+  if ("$1" != "OK")
+    book setword fixchip.book $options:1 pantasksQuality $1
+    exec echo "failure $1 $options:0" >> $tmpline
+  else
+    exec echo "success $1 $options:0" >> $tmpline
+  end 
+  if ($Ndone == $Nhosts)
+    book setword fixchip.book $options:1 pantaskState STOP
+  end
+  book delpage fixhost.book $options:0
+end
+