Index: /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh	(revision 42403)
+++ /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh	(revision 42404)
@@ -3,6 +3,9 @@
 if ($#argv < 1) then
   echo "USAGE: check.rawfix.sh (mode) [options]"
+  echo "USAGE: check.rawfix.sh all"
+  echo "USAGE: check.rawfix.sh stats"
   echo "USAGE: check.rawfix.sh night (YYYYMMDD)"
-  echo "USAGE: check.rawfix.sh new"
+  echo "USAGE: check.rawfix.sh state (state)"
+  echo " NOTE: state = raw, rsync, md5sum.new"
   exit 2
 endif
@@ -61,4 +64,6 @@
   if ($mystate == "rsync") goto good_state
   if ($mystate == "md5sum.new") goto good_state
+  if ($mystate == "md5sum.done") goto good_state
+  if ($mystate == "md5sum.fail") goto good_state
 
   echo "ERROR: unknown mode $mystate"
Index: /trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl	(revision 42404)
+++ /trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl	(revision 42404)
@@ -0,0 +1,108 @@
+#! /usr/bin/env perl
+
+my $DEBUG = 4;
+my $time = time();
+
+use DBI;
+use Carp;
+use Pod::Usage qw( pod2usage );
+
+use strict;
+use warnings FATAL => qw( all );
+use Getopt::Long qw( GetOptions :config auto_help auto_version );
+
+# USAGE: check_md5s_dateobs.pl --dateobs (night)
+
+my ($dateword) = &parse_cmdopts;
+
+my @instlist = glob ("$dateword/*.inst.txt");
+my @md5slist = glob ("$dateword/*.md5s.txt");
+
+foreach my $instfile (@instlist) {
+
+    # find the matching md5sum file for this host
+
+    my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.txt|;
+    # print STDERR "host: $host\n";
+
+    my $md5match = "NONE";
+    foreach my $md5sfile (@md5slist) {
+	
+	my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|;
+	# print STDERR "md5host: $md5shost\n";
+	
+	if ($host ne $md5shost) { next; }
+
+	$md5match = $md5sfile;
+	last;
+    }
+
+    if ($md5match eq "NONE") {
+	print STDERR "missing md5sum file for $instfile\n";
+	next;
+    }
+
+    ## read in the md5sums
+    open (FILE, $md5match);
+    my @md5data = <FILE>;
+    close (FILE);
+
+    ## read in the list of instances
+    open (FILE, $instfile);
+    my @instdata = <FILE>;
+    close (FILE);
+
+    # if these do not match, complain and move on
+    my $N1 = @md5data;
+    my $N2 = @instdata;
+    if ($N1 != $N2) {
+	print STDERR "inconsistent number of lines in $md5match : $N1 vs $N2\n";
+	next;
+    }
+
+    # check for mismatched MD5sums
+    foreach my $line (@md5data) {
+
+	chomp $line;
+	my @words = split (/\s+/,$line);
+
+	# we expect lines of the form: INST STATE MDtru MDobs
+	if (@words != 4) { die "error in MD5 list $md5match, line: $line\n"; }
+
+	my $inst   = $words[0];
+	my $state  = $words[1];
+	my $md5tru = $words[2];
+	my $md5obs = $words[3];
+
+	if ($md5tru ne $md5obs) {
+	    print STDERR "FAIL: $inst $state $md5tru $md5obs\n";
+	}
+    }
+}
+
+sub parse_cmdopts {
+
+    my ($dateobs);
+    GetOptions( 'dateobs=s' => \$dateobs) || pod2usage(2);
+    
+    pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs;
+    
+    # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?)
+
+    my ($date_year, $date_month, $date_day) = split "/",$dateobs;
+    pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day;
+    pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030);
+    pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12);
+    pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31);
+
+    my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day);
+
+    return ($dateword);
+}
+
+sub vsystem {
+    print STDERR "@_\n";
+    #return 0;
+    my $status = system ("@_");
+    return $status;
+}
Index: /trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl	(revision 42403)
+++ /trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl	(revision 42404)
@@ -12,5 +12,5 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version );
 
-# USAGE: fix_chip_locations.pl --dateobs (night)
+# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file)
 
 # we need to select targets for ITC copies of files currently only at ATRC:
Index: /trunk/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh	(revision 42403)
+++ /trunk/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh	(revision 42404)
@@ -14,4 +14,4 @@
 
 # report both the dateobs and the hostnames 
-ls $dirname/*.inst.txt | awk -v dateobs=$dirname -F/ '(NR == 1){print "dateobs md5host"}{print dateobs, $2}' | sed s/.inst.txt// | head 
+ls $dirname/*.inst.txt | awk -v dateobs=$dirname -F/ '(NR == 1){print "dateobs md5host"}{print dateobs, $2}' | sed s/.inst.txt//
 exit 0
Index: /trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl	(revision 42403)
+++ /trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl	(revision 42404)
@@ -27,4 +27,10 @@
 
 if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; }
+if (-e $output) {
+    # check that all instances are in the output file?
+    print "$output exists, skipping\n";
+    exit 0;
+}
+
 open (OUTFILE, ">$output");
 
Index: /trunk/tools/eam/rawfix.20230221/src/gpc1.chips.txt
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/gpc1.chips.txt	(revision 42404)
+++ /trunk/tools/eam/rawfix.20230221/src/gpc1.chips.txt	(revision 42404)
@@ -0,0 +1,60 @@
+XY01
+XY02
+XY03
+XY04
+XY05
+XY06
+XY10
+XY11
+XY12
+XY13
+XY14
+XY15
+XY16
+XY17
+XY20
+XY21
+XY22
+XY23
+XY24
+XY25
+XY26
+XY27
+XY30
+XY31
+XY32
+XY33
+XY34
+XY35
+XY36
+XY37
+XY40
+XY41
+XY42
+XY43
+XY44
+XY45
+XY46
+XY47
+XY50
+XY51
+XY52
+XY53
+XY54
+XY55
+XY56
+XY57
+XY60
+XY61
+XY62
+XY63
+XY64
+XY65
+XY66
+XY67
+XY71
+XY72
+XY73
+XY74
+XY75
+XY76
Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.pt
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/rawfix.pt	(revision 42403)
+++ /trunk/tools/eam/rawfix.20230221/src/rawfix.pt	(revision 42404)
@@ -13,5 +13,10 @@
 
 # use this to find the files in this local directory
-$mypath = `pwd`
+$mypath = /home/panstarrs/eugene/rawfix.20230221
+
+if (not($?LOGDIR)) 
+  $LOGDIR = $mypath/pantasks_logs
+  mkdir $LOGDIR
+end
 
 if (not($?RAWFIX_INIT))
@@ -19,17 +24,4 @@
   book init md5host.book
   $RAWFIX_INIT = 1
-end
-
-macro load.hosts
-  # read the list of possible hosts from the file
-  # for each host, create a variable $host:name = 0
-  # when the host is running, set this to 1
-
-  list allhosts -x "cat allhosts.txt"
-
-  for i 0 $allhosts:n
-    $name = $allhosts:$i
-    $host:$name = 0
-  end
 end
 
@@ -42,4 +34,7 @@
   periods -timeout 30
 
+  stdout $LOGDIR/md5sum.load.stdout.log  
+  stderr $LOGDIR/md5sum.load.stderr.log
+
   # only run one of these tasks at a time
   npending 1
@@ -77,4 +72,7 @@
   periods      -timeout 60
   npending 1
+
+  stdout $LOGDIR/md5sum.loadhosts.stdout.log  
+  stderr $LOGDIR/md5sum.loadhosts.stderr.log
 
   task.exec
@@ -94,4 +92,9 @@
     option $pageName
     # save the page so we can use it below
+
+    # write an entry point in the log for this night
+    sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $DATEOBS
+    date -var mydate
+    exec echo "# start runhosts $mydate" >> $tmpline
 
     command get_hosts_md5s.sh $DATEOBS
@@ -133,4 +136,7 @@
   periods      -timeout 60
 
+  stdout $LOGDIR/md5sum.runhosts.stdout.log  
+  stderr $LOGDIR/md5sum.runhosts.stderr.log
+
   task.exec
     # if we are unable to run the 'exec', use a long retry time
@@ -152,24 +158,25 @@
     # save DATEOBS so we can find the md5sum.book entry
 
-    # XXX test: use test host
-    $TESTHOST = ipp060
-    host -required $TESTHOST
-    if (not($?host:$TESTHOST)) 
-      echo "host not defined, creating : $TESTHOST"
-      $host:$TESTHOST = 0
+    # XXX TEST: $USE_HOST = ipp060
+
+    $USE_HOST = $MD5HOST
+    host -required $USE_HOST
+    ## check if this host needs to be launched
+    if (not($?host:$USE_HOST)) 
+      echo "host not defined, creating : $USE_HOST"
+      $host:$USE_HOST = 0
     end
-    if (not($host:$TESTHOST)) 
-      control host add $TESTHOST
-      $host:$TESTHOST = 1
+    if (not($host:$USE_HOST)) 
+      control host add $USE_HOST
+      $host:$USE_HOST = 1
     end
-    # host -required $MD5HOST 
-
-    ## check if this host needs to be launched
-
+
+    ## even when testing, use the original hostname here:
     $input = $mypath/$DATEOBS/$MD5HOST.inst.txt
 
     # need to make sure this will work on the ipp b-nodes:
-    # command $mypath/get_md5s_instances.pl --hostname $MD5HOST --input $input
-    command $mypath/get_md5s_instances.pl --hostname $TESTHOST --input $input
+    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
+    periods -exec 0.05
+    command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input
   end
 
@@ -203,4 +210,7 @@
   periods      -timeout 60
   npending 1
+
+  stdout $LOGDIR/md5sum.clean.stdout.log  
+  stderr $LOGDIR/md5sum.clean.stderr.log
 
   task.exec
@@ -214,14 +224,11 @@
     # look for completed nights in md5sum.book (pantaskState == DONE)
     book getpage md5sum.book 0 -var pageName -key pantaskState DONE
-    echo "checking this page: $pageName"
     if ("$pageName" == "NULL") break
 
-    echo "cleaning this page: $pageName"
-    book listpage md5sum.book $pageName
+    # book listpage md5sum.book $pageName
 
     # book setword md5sum.book $pageName pantaskState RUN
     book getword md5sum.book $pageName dateobs -var DATEOBS
     book getword md5sum.book $pageName pantasksQuality -var QUALITY
-    echo "cleaning $pageName, quality: $QUALITY"
 
     option $pageName
@@ -229,5 +236,4 @@
 
     if ("$QUALITY" == "OK")
-       
       command update.rawfix.sh $DATEOBS md5sum.done 
     else
@@ -275,4 +281,8 @@
   if ("$1" != "OK")
     book setword md5sum.book $options:1 pantasksQuality $1
+    sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $options:1
+    exec echo "failure $1 $options:0" >> $tmpline
+  else
+    exec echo "success $1 $options:0" >> $tmpline
   end 
   if ($Ndone == $Nhosts)
Index: /trunk/tools/eam/rawfix.20230221/src/update.bnodes.sh
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/update.bnodes.sh	(revision 42403)
+++ /trunk/tools/eam/rawfix.20230221/src/update.bnodes.sh	(revision 42404)
@@ -6,5 +6,5 @@
 endif
 
-set options = "-auvn"
+set options = "-auvn --exclude=trash"
 if ($2 == "commit") set options = "-auv"