Changeset 42449 for trunk/tools

Timestamp:

Apr 13, 2023, 9:30:40 AM (3 years ago)

Author:

eugene

Message:

split fix_chip_locations into parallel processes by chip

Location:

trunk/tools/eam/rawfix.20230221/src

Files:

• advance.rawfix.pl (modified) (2 diffs)
• finish.fixchip.sh (added)
• fix_chip_locations.pl (modified) (7 diffs)
• get_hosts_fixchip.sh (added)
• queue.rawfix.sh (modified) (1 diff)
• rawfix.fixchip.pt (modified) (8 diffs)
• rawfix.md5sum.pt (modified) (1 diff)
• update_chip_stats.pl (added)

trunk/tools/eam/rawfix.20230221/src/advance.rawfix.pl

@@ -144 +144 @@
     my $Nfail    = $values[14];
     
+    # in state of 'new', we do not care what the values of the stat fields are.
+    # they may be NULL, so do not yet try to do math on them here
+    if ($state eq "new")                                              {           &update_night_state ($dateobs, "ckchip.new", 0);     return "NEW"; }
+
     my $Ndone = $Ngood + $Ndead + $Ngone + $Nfail;
     my $Nwait = $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3;
@@ -151 +155 @@
     # check for completion here first (otherwise, advance fixchip.done below)
     if (($state eq "fixchip.done") and ($iter > 0) and ($Nwait == 0)) {           &update_night_state ($dateobs, "DONE");              return "DONE"; }
-    if ($state eq "new")                                              {           &update_night_state ($dateobs, "ckchip.new", 0);     return "NEW"; }
     if ($state eq "ckchip.done")                                      {           &update_night_state ($dateobs, "fixchip.new");       return "NEW"; }
     if ($state eq "fixchip.done")                                     {           &update_night_state ($dateobs, "md5sum.new");        return "NEW"; }

trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl

@@ -12 +12 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version );
 
-# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) --stage (0,1,etc)
+# USAGE: fix_chip_locations.pl --dateobs (night) --stage (0,1,etc) [--chiplist (file) | --chipID (ID)]
 
 # we need to select targets for ITC copies of files currently only at ATRC:
@@ -18 +18 @@
 # for each chip, target a single host?
 
-my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts;
+my ($fulldate, $dateword, $chiplist, $chipID, $stage, $topdir) = &parse_cmdopts;
 my ($neb_dbh, $raw_dbh) = &parse_db_config ("$topdir/nebulous.config");
-
-# read the list of chip_ids:
-my @chip_ids = &load_chip_list ($chiplist);
 
 # choose ITC machines on which to place backup copies:
 my @vol_copy = &get_neb_volumes_copy; # array of neb hosts to copy data to at ITC
 my %vol_good = &get_neb_volumes_good; # hash of valid neb hosts
-my $volnum = 0;
 
 my $Ngood = 0;
@@ -41 +37 @@
 my $Ncul3 = 0;
 my $Nfail = 0;
+
+my $Nchip = 0; # counts the number of instances
 
 # load the lists of good and fail chips
@@ -69 +67 @@
 }
 
-my $Nchip = 0;
-
+# read the list of chip_ids:
+my @chip_ids = &load_chip_list ($chiplist);
+
+my $volnum = 0; # select the volume to be used from the list of @vol_copy
+
+# loop over all chip_ids so we can distribute the target volumes
 foreach my $chip_id (@chip_ids) {
+    if (not defined $chipID or ($chipID eq $chip_id)) {
+        &fix_chip_locations_chipID ($chip_id, $volnum);
+    }
+    $volnum ++;
+    if ($volnum > @vol_copy) { $volnum = 0; }
+}
+
+my $Ntotal = $Ngood + $Ngone + $Ndead + $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3 + $Nfail;
+if ($Ntotal != $Nchip) { die "ERROR: inconsistent count of chips\n"; } 
+
+if (not defined $chipID) {
+    # update the stats in the database
+    my $sql_update_stats = "UPDATE nights set";
+    $sql_update_stats .= "  Nchip = $Nchip";
+    $sql_update_stats .= ", Ngood = $Ngood";
+    $sql_update_stats .= ", Ndead = $Ndead";
+    $sql_update_stats .= ", Ngone = $Ngone";
+    $sql_update_stats .= ", Nsing = $Nsing";
+    $sql_update_stats .= ", Ndups = $Ndups";
+    $sql_update_stats .= ", Nsbck = $Nsbck";
+    $sql_update_stats .= ", Nxbck = $Nxbck";
+    $sql_update_stats .= ", Ncul1 = $Ncul1";
+    $sql_update_stats .= ", Ncul2 = $Ncul2";
+    $sql_update_stats .= ", Ncul3 = $Ncul3";
+    $sql_update_stats .= ", Nfail = $Nfail";
+    $sql_update_stats .= " WHERE dateobs = $dateword";
+    
+    my $query = $raw_dbh->prepare($sql_update_stats);
+    $query->execute () or die "invalid response from RAW database\n";
+} else {
+
+    open (OUTFILE, ">$topdir/$dateword/$chipID.stats.$stage.txt");
+
+    print OUTFILE "Nchip: $Nchip\n";
+    print OUTFILE "Ngood: $Ngood\n";
+    print OUTFILE "Ngone: $Ngone\n";
+    print OUTFILE "Ndead: $Ndead\n";
+    print OUTFILE "Nsing: $Nsing\n";
+    print OUTFILE "Ndups: $Ndups\n";
+    print OUTFILE "Nsbck: $Nsbck\n";
+    print OUTFILE "Nxbck: $Nxbck\n";
+    print OUTFILE "Ncul1: $Ncul1\n";
+    print OUTFILE "Ncul2: $Ncul2\n";
+    print OUTFILE "Ncul3: $Ncul3\n";
+    print OUTFILE "Nfail: $Nfail\n";
+
+    close (OUTFILE);
+}
+
+exit 0;
+
+sub fix_chip_locations_chipID {
+
+    my $chip_id = $_[0];
+    my $volnum  = $_[1];
 
     # read from chip_id file, update to next stage number
@@ -216 +273 @@
         }
     }
-    $volnum ++;
-}
-
-print "Nchip: $Nchip\n";
-print "Ngood: $Ngood\n";
-print "Ngone: $Ngone\n";
-print "Ndead: $Ndead\n";
-print "Nsing: $Nsing\n";
-print "Ndups: $Ndups\n";
-print "Nsbck: $Nsbck\n";
-print "Nxbck: $Nxbck\n";
-print "Ncul1: $Ncul1\n";
-print "Ncul2: $Ncul2\n";
-print "Ncul3: $Ncul3\n";
-print "Nfail: $Nfail\n";
-
-my $Ntotal = $Ngood + $Ngone + $Ndead + $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3 + $Nfail;
-if ($Ntotal != $Nchip) { die "ERROR: inconsistent count of chips\n"; } 
-
-# update the stats in the database
-my $sql_update_stats = "UPDATE nights set";
-$sql_update_stats .= "  Nchip = $Nchip";
-$sql_update_stats .= ", Ngood = $Ngood";
-$sql_update_stats .= ", Ndead = $Ndead";
-$sql_update_stats .= ", Ngone = $Ngone";
-$sql_update_stats .= ", Nsing = $Nsing";
-$sql_update_stats .= ", Ndups = $Ndups";
-$sql_update_stats .= ", Nsbck = $Nsbck";
-$sql_update_stats .= ", Nxbck = $Nxbck";
-$sql_update_stats .= ", Ncul1 = $Ncul1";
-$sql_update_stats .= ", Ncul2 = $Ncul2";
-$sql_update_stats .= ", Ncul3 = $Ncul3";
-$sql_update_stats .= ", Nfail = $Nfail";
-$sql_update_stats .= " WHERE dateobs = $dateword";
-
-my $query = $raw_dbh->prepare($sql_update_stats);
-$query->execute () or die "invalid response from RAW database\n";
-
-exit 0;
+}
 
 sub get_new_instance {
@@ -412 +431 @@
 sub parse_cmdopts {
 
-    my ($dateobs, $chiplist, $stage, $topdir);
-    GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2);
+    my ($dateobs, $chiplist, $chipID, $stage, $topdir);
+    GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'chipID=s' => \$chipID, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2);
     
     pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs;
-    pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist;
     pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage;
     pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir;
-    
+    pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist;
+
     # check that stage is an integer
     unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; }
@@ -435 +454 @@
     my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day);
 
-    return ($fulldate, $dateword, $chiplist, $stage, $topdir);
+    return ($fulldate, $dateword, $chiplist, $chipID, $stage, $topdir);
 }
 

trunk/tools/eam/rawfix.20230221/src/queue.rawfix.sh

@@ -29 +29 @@
 if ($mode == "night") then
   ## XXX validate the format (YYYYMMDD)?
-
- XXXXX:
-
   set mynight = $2
   

trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

@@ -9 +9 @@
 if (not($?RAWFIX_FIXCHIP_INIT))
   book init fixchip.book
+  book init fixhost.book
   $RAWFIX_FIXCHIP_INIT = 1
 end
@@ -27 +28 @@
 
   task.exec
-    # command check.rawfix.sh state ckchip.done
     command check.rawfix.sh state fixchip.new 
   end
@@ -53 +53 @@
 end
 
-task nights.fixchip.run
+# output: fixhost.book
+task nights.fixchip.loadhosts
   periods      -poll $RPOLL
   periods      -exec $REXEC
-  periods      -timeout 1800
-  host         -required $HOST_FIXCHIP
+  periods      -timeout 60
+  host          local
   npending 1
+  # note: the tasks which return their results via stdout
+  # seem to fail if run remotely.  
+
+  stdout $LOGDIR/fixchip.loadhosts.stdout.log  
+  stderr $LOGDIR/fixchip.loadhosts.stderr.log
 
   task.exec
@@ -76 +82 @@
     book getword fixchip.book $pageName iter    -var ITER
     option $pageName
-
-    stdout $pageName/log.fixchip.v$ITER
-    stderr $pageName/log.fixchip.v$ITER
+    # save the page so we can use it below
+
+    # write an entry point in the log for this night
+    sprintf tmpline "%s/%s/log.fixchip.runhosts.v%d" $mypath $DATEOBS $ITER
+    date -var mydate
+    exec echo "# start runhosts $mydate" >> $tmpline
+
+    # the command returns these fields: dateobs stage fixhost chipID
+    # NOTE: stage == ITER
+    command $mypath/get_hosts_fixchip.sh $mypath $DATEOBS $ITER
+  end
+
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    queueprint stdout
+    queuesize stdout -var Nhosts
+    queue2book -raw-columns -key dateobs:chipID -uniq -setword pantaskState INIT stdout fixhost.book
+    book setword fixchip.book $options:0 NHOST {$Nhosts - 1}
+    book setword fixchip.book $options:0 NDONE 0
+    book list
+    book listbook fixchip.book
+    # use the queuesize (Nhosts) to track how many jobs are done / to be done
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+    book setword fixchip.book $options:0 pantasksQuality FAIL
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword fixchip.book $options:0 pantasksQuality CRASH
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword fixchip.book $options:0 pantasksQuality TIMEOUT
+  end
+end
+
+task nights.fixchip.run
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 1800
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages fixhost.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new nights in fixhost.book (pantaskState == INIT)
+    book getpage fixhost.book 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword fixhost.book $pageName pantaskState RUN
+    book getword fixhost.book $pageName dateobs -var DATEOBS
+    book getword fixhost.book $pageName iter    -var ITER
+    book getword fixhost.book $pageName fixhost -var FIXHOST
+    book getword fixhost.book $pageName chipID  -var CHIP_ID
+    option $pageName $DATEOBS
+
+    stdout $pageName/log.fixchip.run.v$ITER
+    stderr $pageName/log.fixchip.run.v$ITER
 
     substr $DATEOBS 0 4 YEAR
@@ -85 +157 @@
     sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
 
-    command $mypath/fix_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage $ITER
-  end
-
-  ## XX delete the page from the book if done?
-  task.exit 0
+    host -required $FIXHOST
+    ## check if this host needs to be launched
+    if (not($?host:$FIXHOST)) 
+      echo "host not defined, creating : $FIXHOST"
+      $host:$FIXHOST = 0
+    end
+    if (not($host:$FIXHOST)) 
+      control host add $FIXHOST
+      $host:$FIXHOST = 1
+    end
+
+    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
+    periods -exec 0.05
+    command $mypath/fix_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --chipID $CHIP_ID --stage $ITER
+  end
+
+  task.exit 0
+    nights.fixchip.exit OK
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    nights.fixchip.exit FAIL
+    showcommand failure
+  end
+
+  task.exit    crash
+    nights.fixchip.exit CRASH
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    nights.fixchip.exit TIMEOUT
+    showcommand timeout
+  end
+end
+
+# output: fixhost.book
+task nights.fixchip.clean
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+  npending 1
+
+  stdout $LOGDIR/fixchip.clean.stdout.log  
+  stderr $LOGDIR/fixchip.clean.stderr.log
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages fixchip.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for completed nights in fixchip.book (pantaskState == STOP)
+    book getpage fixchip.book 0 -var pageName -key pantaskState STOP
+    if ("$pageName" == "NULL") break
+
+    # book listpage fixchip.book $pageName
+
+    # book setword fixchip.book $pageName pantaskState RUN
+    book getword fixchip.book $pageName dateobs -var DATEOBS
+    book getword fixchip.book $pageName pantasksQuality -var QUALITY
+
+    option $pageName
+    # save the page so we can use it below
+
+    if ("$QUALITY" == "OK")
+      command update.rawfix.sh $DATEOBS fixchip.done 
+    else
+      command update.rawfix.sh $DATEOBS fixchip.fail 
+    end
+  end
+
+  task.exit 0
+    # now that the entry is set to done in the db, remove the pantasks book entry
+    echo "cleaning this page: $options:0"
     book setword fixchip.book $options:0 pantaskState DONE
-    exec update.rawfix.sh $options:0 fixchip.done
   end
 
@@ -98 +244 @@
     showcommand failure
     book setword fixchip.book $options:0 pantasksQuality FAIL
-    exec update.rawfix.sh $options:0 fixchip.fail
   end
 
@@ -104 +249 @@
     showcommand crash
     book setword fixchip.book $options:0 pantasksQuality CRASH
-    exec update.rawfix.sh $options:0 fixchip.fail
   end
 
@@ -111 +255 @@
     showcommand timeout
     book setword fixchip.book $options:0 pantasksQuality TIMEOUT
-    exec update.rawfix.sh $options:0 fixchip.fail
-  end
-end
-
+  end
+end
+
+######################### UTILITIES #################################
+
+macro nights.fixchip.exit
+  if ($0 != 2)
+    echo "USAGE: nights.fixchip.exit (quality)"
+    break
+  end
+
+  book setword fixhost.book $options:0 pantaskState DONE
+  book getword fixchip.book $options:1 NHOST -var Nhosts
+  book getword fixchip.book $options:1 NDONE -var Ndone
+  book getword fixchip.book $options:1 stage -var ITER
+  $Ndone ++
+  book setword fixchip.book $options:1 NDONE $Ndone
+  if ("$1" != "OK")
+    book setword fixchip.book $options:1 pantasksQuality $1
+    sprintf tmpline "%s/%s/log.fixchip.runhosts.v%d" $mypath $options:1 $ITER
+    exec echo "failure $1 $options:0" >> $tmpline
+  else
+    exec echo "success $1 $options:0" >> $tmpline
+  end 
+  if ($Ndone == $Nhosts)
+    book setword fixchip.book $options:1 pantaskState STOP
+  end
+  book delpage fixhost.book $options:0
+end
+

trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt

@@ -127 +127 @@
 # runs on the hosts identified by get_hosts_md5s.sh
 # output: md5host.book
-task nights.md5sum.runhosts
+task nights.md5sum.run
 
   periods      -poll $RPOLL