Index: /trunk/tools/eam/rawfix.20230221/src/advance.rawfix.pl
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/advance.rawfix.pl	(revision 42451)
+++ /trunk/tools/eam/rawfix.20230221/src/advance.rawfix.pl	(revision 42452)
@@ -77,5 +77,8 @@
 
     # select and advance nights not yet running
-    $sql_get_run = "SELECT dateobs from nights where state = 'new'";
+#   $sql_get_run = "SELECT dateobs from nights where state = 'new'";
+
+    $sql_get_run = "SELECT dateobs FROM (SELECT dateobs, rand() AS myrand FROM nights WHERE (state = 'new')) AS tmp ORDER BY myrand";
+
     $runref   = $raw_dbh->selectall_arrayref( $sql_get_run );
     @runarray = @$runref;
Index: /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl	(revision 42451)
+++ /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl	(revision 42452)
@@ -196,8 +196,14 @@
 	if ($Nvalues != 3) { print "ERROR: invalid response?\n"; next; }
 	
+	# data_state and possibly md5sum can be NULL, need to handle specially
+	my $dstate = $values[1]; if (not defined $dstate) { $dstate = "NULL"; } 
+	my $md5sum = $values[2]; if (not defined $md5sum) { $md5sum = "XXXX"; } 
+	
 	push (@nebarray, $values[0]);
-	push (@ds_array, $values[1]);
-	push (@md_array, $values[2]);
-	if ($DEBUG > 4) { print "$values[0]\n $values[1]\n $values[2]\n"; }
+	push (@md_array, $md5sum);
+	push (@ds_array, $dstate);
+
+	if ($DEBUG > 5) { print "V0: $values[0]\n"; print "V1: $values[1]\n"; print "V2: $values[2]\n"; }
+	if ($DEBUG > 4) { print "PRINT: $values[0]\n $dstate\n $md5sum\n"; }
     }
 
Index: /trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl	(revision 42451)
+++ /trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl	(revision 42452)
@@ -16,6 +16,6 @@
 my ($dateword, $stage, $topdir) = &parse_cmdopts;
 
-my @instlist = glob ("$topdir/$dateword/*.inst.$stage.txt");
-my @md5slist = glob ("$topdir/$dateword/*.md5s.txt");
+my $instlist_r = &load_instlist_filenames ($topdir, $dateword, $stage);
+my $md5slist_r = &load_md5slist_filenames ($topdir, $dateword, $stage, $instlist_r);
 
 my $Nfail = 0;
@@ -23,38 +23,18 @@
 my %goodchips;
 
-foreach my $instfile (@instlist) {
-
-    # find the matching md5sum file for this host
-
-    my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|;
-    # print STDERR "host: $host\n";
-
-    my $md5match = "NONE";
-    foreach my $md5sfile (@md5slist) {
-	
-	my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|;
-	# print STDERR "md5host: $md5shost\n";
-	
-	if ($host ne $md5shost) { next; }
-
-	$md5match = $md5sfile;
-	last;
-    }
-
-    if ($md5match eq "NONE") {
-	print STDOUT "missing md5sum file for $instfile\n";
-	$Nfail ++;
-	next;
-    }
+# make a list of the master instance files:
+
+foreach my $instfile (@$instlist_r) {
+
+    # print "inst: $instfile\n";
+
+    my $md5sfile = $md5slist_r->{$instfile};
 
     ## read in the md5sums
-    open (FILE, $md5match);
+    open (FILE, $md5sfile);
     my @md5data = <FILE>;
     close (FILE);
 
-    ## read in the list of instances
-    open (FILE, $instfile);
-    my @instdata = <FILE>;
-    close (FILE);
+    my $instdata_r = &load_instance_list ($topdir, $dateword, $stage, $instfile);
 
     # if these do not match, complain and move on
@@ -62,14 +42,16 @@
     # may have been culled during this process (but the MD5s stay behind)
     my $N1 = @md5data;
-    my $N2 = @instdata;
+    my $N2 = @$instdata_r;
     if ($N1 < $N2) {
-	print STDOUT "not enough MD5s $md5match : $N1 vs $N2\n";
+	print STDOUT "not enough MD5s $md5sfile : $N1 vs $N2\n";
 	$Nfail ++;
 	next;
+    } else {
+	# print STDOUT "checking MD5s $md5sfile : $N1 vs $N2\n";
     }
 
     # set up an empty hash for the instance names:
     my %instfound = ();
-    foreach my $line (@instdata) {
+    foreach my $line (@$instdata_r) {
 	chomp $line;
 	my @words = split (/\s+/,$line);
@@ -86,5 +68,5 @@
 
 	# we expect lines of the form: INST STATE MDtru MDobs
-	if (@words != 4) { die "error in MD5 list $md5match, line: $line\n"; }
+	if (@words != 4) { die "error in MD5 list $md5sfile, line: $line\n"; }
 
 	my $inst   = $words[0];
@@ -136,4 +118,113 @@
     print STDOUT "SUCCESS: $Nfail errors\n";
     exit 0;
+}
+
+# use global $topdir, $dateword, $myhost, $stage
+sub load_instance_list {
+    my $topdir   = $_[0];
+    my $dateword = $_[1];
+    my $stage    = $_[2];
+    my $instfile = $_[3];
+
+    my @instdata_list;
+
+    # load the master list
+    &load_instance_list_onefile ($instfile, \@instdata_list);
+
+    # there may be additional instances in files with names like HOSTNAME.XYnn.inst.STAGE 
+    my ($myhost) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|;
+    my @instlist = glob ("$topdir/$dateword/$myhost.XY??.inst.$stage.txt");
+    foreach my $infile  (@instlist) {
+	&load_instance_list_onefile ($infile, \@instdata_list);
+    }
+    return (\@instdata_list);
+}
+
+# read the instance info
+sub load_instance_list_onefile {
+    my $infile = $_[0];
+    my $instdata_list_r = $_[1]; # reference to the file data
+
+    open(FILE,$infile) or die "cannot open file $infile\n"; 
+    my @lines = <FILE>;
+    close (FILE);
+
+    foreach my $line (@lines) {
+	chomp $line;
+	if ($line =~ /^\s*#/) { next; } # skip lines with leading #
+	push (@$instdata_list_r, $line);
+    }
+}
+
+# return only the master instlist filenames (ignore ones with the chipnames in the name)
+sub load_instlist_filenames {
+    
+    my $topdir   = $_[0];
+    my $dateword = $_[1];
+    my $stage    = $_[2];
+
+    my @instlist_all = glob ("$topdir/$dateword/*.inst.$stage.txt");
+
+    my @instlist = ();
+
+    foreach my $instfile (@instlist_all) {
+
+	# skip the per-chip inst files (these should probably be merged into the master)
+	if ($instfile =~ m|XY[0-9][0-9].inst|) { next; }
+
+	push (@instlist, $instfile);
+    }
+    return (\@instlist);
+}
+
+# return a hash of the matched md5 file for each inst file
+sub load_md5slist_filenames {
+
+    my $topdir     = $_[0];
+    my $dateword   = $_[1];
+    my $stage      = $_[2];
+    my $instlist_r = $_[3];
+
+    my @md5slist = glob ("$topdir/$dateword/*.md5s.txt");
+
+    my %md5shash;
+
+    my $Nfail = 0;
+
+    foreach my $instfile (@$instlist_r) {
+
+	# find the matching md5sum file for this host
+
+	my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|;
+	# print STDERR "host: $host\n";
+
+	my $md5match = "NONE";
+	foreach my $md5sfile (@md5slist) {
+	
+	    my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|;
+	
+	    if ($host ne $md5shost) { next; }
+
+	    # print STDERR "md5host: $md5shost : $md5sfile = $instfile\n";
+
+	    $md5match = $md5sfile;
+	    last;
+	}
+
+	if ($md5match eq "NONE") {
+	    print STDOUT "missing md5sum file for $instfile\n";
+	    $Nfail ++;
+	    next;
+	}
+	$md5shash{$instfile} = $md5match;
+
+	# print "match: $instfile = $md5match\n";
+
+    }
+    if ($Nfail > 0) { 
+	print STDOUT "ERROR: $Nfail errors\n";
+	exit 1;
+    }
+    return (\%md5shash);
 }
 
Index: /trunk/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh	(revision 42451)
+++ /trunk/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh	(revision 42452)
@@ -21,4 +21,4 @@
 
 # report both the dateobs and the hostnames 
-ls $topdir/$dateobs/*.inst.$stage.txt | awk -v dateobs=$dateobs -v stage=$stage -F/ '(NR == 1){print "dateobs stage md5host"}{print dateobs, stage, $NF}' | sed s/.inst.$stage.txt//
+ls $topdir/$dateobs/*.inst.$stage.txt | grep -v "XY...inst" | awk -v dateobs=$dateobs -v stage=$stage -F/ '(NR == 1){print "dateobs stage md5host"}{print dateobs, stage, $NF}' | sed s/.inst.$stage.txt//
 exit 0
Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.advance.pt
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/rawfix.advance.pt	(revision 42451)
+++ /trunk/tools/eam/rawfix.20230221/src/rawfix.advance.pt	(revision 42452)
@@ -1,4 +1,4 @@
 
-$MAX_ACTIVE_NIGHTS = 5
+$MAX_ACTIVE_NIGHTS = 10
 
 ## regularly run the command advance.rawfits.sh
@@ -8,4 +8,5 @@
   periods -timeout 30
   host local
+  active true
 
   stdout $LOGDIR/advance.log  
@@ -39,2 +40,16 @@
   end
 end
+
+macro nights.advance.on
+  task nights.advance
+    active true
+  end
+end
+
+macro nights.advance.off
+  task nights.advance
+    active false
+  end
+end
+
+  
Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt	(revision 42451)
+++ /trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt	(revision 42452)
@@ -12,4 +12,6 @@
   $RAWFIX_FIXCHIP_INIT = 1
 end
+
+$MAX_FIXHOST = 2
 
 # select nights in state fixchip.new
@@ -97,11 +99,11 @@
     # convert the stdout 'queue' to entries in a book
     # fields in the table will become words in the page
-    queueprint stdout
+    # queueprint stdout
     queuesize stdout -var Nhosts
     queue2book -raw-columns -key dateobs:chipID -uniq -setword pantaskState INIT stdout fixhost.book
     book setword fixchip.book $options:0 NHOST {$Nhosts - 1}
     book setword fixchip.book $options:0 NDONE 0
-    book list
-    book listbook fixchip.book
+    # book list
+    # book listbook fixchip.book
     # use the queuesize (Nhosts) to track how many jobs are done / to be done
   end
@@ -163,7 +165,7 @@
       $host:$FIXHOST = 0
     end
-    if (not($host:$FIXHOST)) 
+    if ($host:$FIXHOST < $MAX_FIXHOST) 
       control host add $FIXHOST
-      $host:$FIXHOST = 1
+      $host:$FIXHOST ++
     end
 
Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt	(revision 42451)
+++ /trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt	(revision 42452)
@@ -12,4 +12,6 @@
   $RAWFIX_MD5SUM_INIT = 1
 end
+
+$MAX_MD5HOST = 2
 
 # select nights which are ready for md5sum calcs
@@ -97,11 +99,11 @@
     # convert the stdout 'queue' to entries in a book
     # fields in the table will become words in the page
-    queueprint stdout
+    # queueprint stdout
     queuesize stdout -var Nhosts
     queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host.book
     book setword md5sum.book $options:0 NHOST {$Nhosts - 1}
     book setword md5sum.book $options:0 NDONE 0
-    book list
-    book listbook md5sum.book
+    # book list
+    # book listbook md5sum.book
     # use the queuesize (Nhosts) to track how many jobs are done / to be done
   end
@@ -166,7 +168,7 @@
       $host:$USE_HOST = 0
     end
-    if (not($host:$USE_HOST)) 
+    if ($host:$USE_HOST < $MAX_MD5HOST) 
       control host add $USE_HOST
-      $host:$USE_HOST = 1
+      $host:$USE_HOST ++
     end