Index: /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh	(revision 42399)
+++ /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh	(revision 42400)
@@ -16,5 +16,26 @@
 set DBPASS = "dvo"
 
-set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME"
+# set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME"
+set DBOPTS = "-B -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME"
+
+if ($mode == "all") then
+  if ($#argv != 1) then
+    echo "USAGE: check.rawfix.sh all"
+    exit 2
+  endif
+
+  mysql $DBOPTS -e "select dateobs, state from nights"
+  exit 0
+endif
+
+if ($mode == "stats") then
+  if ($#argv != 1) then
+    echo "USAGE: check.rawfix.sh stats"
+    exit 2
+  endif
+
+  mysql $DBOPTS -e "select count(dateobs) Nnights, state from nights group by state"
+  exit 0
+endif
 
 if ($mode == "night") then
@@ -29,6 +50,21 @@
 endif
 
-if ($mode == "new") then
-  mysql $DBOPTS -e "select dateobs from nights where (state = 'new')"
+if ($mode == "state") then
+  if ($#argv != 2) then
+    echo "USAGE: check.rawfix.sh state (state)"
+    exit 2
+  endif
+
+  set mystate = $2
+
+  if ($mystate == "new") goto good_state
+  if ($mystate == "rsync") goto good_state
+  if ($mystate == "md5sum.new") goto good_state
+
+  echo "ERROR: unknown mode $mystate"
+  exit 2
+
+  good_state:
+  mysql $DBOPTS -e "select dateobs from nights where (state = '$mystate')"
   exit 0
 endif
Index: /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl	(revision 42399)
+++ /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl	(revision 42400)
@@ -40,5 +40,10 @@
 
     # get the list of chip neb paths for one chip
-    my @neb_paths = &get_chip_paths ($fulldate, $chip_id);
+    my ($neb_paths_r, $ds_array_r, $md_array_r) = &get_chip_paths ($fulldate, $chip_id);
+
+    # I think this makes a copy of the arrays, which is inefficient
+    # my @neb_paths = @$neb_paths_r;
+    # my @ds_array  = @$ds_array_r;
+    # my @md_array  = @$md_array_r;
 
     # reset the hostcount for this chip
@@ -46,5 +51,11 @@
 
     # find the neb instances for each chip nebulous entry
-    foreach my $nebname (@neb_paths) {
+
+    for (my $i = 0; $i < @$neb_paths_r; $i++) {
+	my $nebname = $neb_paths_r->[$i]; # equivalent to ${$neb_paths_r}[$i]
+	my $data_st = $ds_array_r->[$i];
+	my $md5_sum = $md_array_r->[$i];
+
+	# print STDERR "nebname: $nebname, dstate: $data_st, md5: $md5_sum\n";
 
 	# get info for all instances of this chip
@@ -54,5 +65,5 @@
 
 	# determine the status of the instances
-	my ($Nmain, $Nback, $Ndead) = &check_instance_xattr ($inforows, $ext_id);
+	my ($Nmain, $Nback, $Ndead) = &check_instance_xattr ($inforows, $ext_id, $data_st, $md5_sum);
 
 	&check_instance_state ($outfh, $ext_id, $Nmain, $Nback, $Ndead);
@@ -130,7 +141,7 @@
     my $chip_id = $_[1];
 
-    my @nebarray;
-
-    my $sql_chip_paths = "SELECT uri from rawImfile WHERE class_id = '$chip_id' and dateobs > '$fulldate,00:00:00' and dateobs < '$fulldate,23:59:59'";
+    my (@nebarray, @ds_array, @md_array);
+
+    my $sql_chip_paths = "SELECT uri, data_state, md5sum from rawImfile WHERE class_id = '$chip_id' and dateobs > '$fulldate,00:00:00' and dateobs < '$fulldate,23:59:59'";
     if ($DEBUG > 5) { print "sql: $sql_chip_paths\n"; }
 
@@ -145,12 +156,14 @@
 	# there should only be a single value (only uri selected above)
 	my $Nvalues = @values;
-	if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; }
-	if ($Nvalues  > 1) { print "ERROR: too many uri entries?\n"; next; }
+	if ($Nvalues != 3) { print "ERROR: invalid response?\n"; next; }
 	
 	push (@nebarray, $values[0]);
-	if ($DEBUG > 4) { print "$values[0]\n"; }
-    }
-    
-    return @nebarray;
+	push (@ds_array, $values[1]);
+	push (@md_array, $values[2]);
+	if ($DEBUG > 4) { print "$values[0]\n $values[1]\n $values[2]\n"; }
+    }
+
+    # return references so we can keep the three arrays distinct
+    return (\@nebarray, \@ds_array, \@md_array);
 }
 
@@ -177,7 +190,9 @@
 sub check_instance_xattr {
 
-    my $ext_id = $_[1];
-
-    my $rowref = $_[0];
+    my $rowref  = $_[0];
+    my $ext_id  = $_[1];
+    my $data_st = $_[2];
+    my $md5_sum = $_[3];
+
     my @inforows = @$rowref;
     my $Nrows    = @inforows;
@@ -218,5 +233,5 @@
 	}
 	my $fh = $hostfile{$hostname};
-	print $fh "$instance\n";
+	print $fh "$instance $data_st $md5_sum\n";
 
 	if ($xattr == 0) { $Nmain ++; }
Index: /trunk/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh	(revision 42400)
+++ /trunk/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh	(revision 42400)
@@ -0,0 +1,17 @@
+#!/bin/csh
+
+if ($#argv != 1) then
+  echo "USAGE: get_hosts_md5s.sh (dateobs)"
+  exit 2
+endif
+
+set dirname = $1
+
+if (! -d $dirname) then
+  echo "ERROR: missing dateobs directory $dirname"
+  exit 3
+endif
+
+# report both the dateobs and the hostnames 
+ls $dirname/*.inst.txt | awk -v dateobs=$dirname -F/ '(NR == 1){print "dateobs md5host"}{print dateobs, $2}' | sed s/.inst.txt// | head 
+exit 0
Index: /trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl	(revision 42399)
+++ /trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl	(revision 42400)
@@ -21,5 +21,5 @@
 
 # read the list of instances:
-my @ins_list = &load_instance_list ($input);
+my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ($input);
 
 my $output = $input;
@@ -29,20 +29,26 @@
 open (OUTFILE, ">$output");
 
-foreach my $instance (@ins_list) {
+# foreach my $instance (@$ins_list_r) {
+for (my $i = 0; $i < @$ins_list_r; $i ++) {
+
+    my $instance   = $ins_list_r->[$i];
+    my $data_state = $dst_list_r->[$i];
+    my $md5sum_tru = $md5_list_r->[$i];
 
     my $filename = $instance;
     $filename =~ s|file://||;
-    # print "filename: $filename\n";
 
-    my $line = `md5sum $filename`;
+    my $line;
+    if (0) {
+	$line = "fakevalue $filename"; # XXX TEST: disable calculation:
+    } else {
+	$line = `md5sum $filename`;
+    }
 
     my ($md5sum, $rawfile) = split (" ", $line);
-    # print "md5sum: $md5sum\n";
-
-    print OUTFILE "$instance $md5sum\n";
+    print OUTFILE "$instance $data_state $md5sum_tru $md5sum\n";
 }
 close OUTFILE;
-
-die "all done";
+exit 0;
 
 # read the list of chip_ids:
@@ -50,5 +56,5 @@
     my $infile = $_[0];
 
-    my @ins_list;
+    my (@ins_list, @dst_list, @md5_list);
 
     open(FILE,$infile) or die "cannot open file $infile\n"; 
@@ -61,10 +67,12 @@
 	my @words = split (/\s+/,$line);
 
-	if (@words != 1) { die "error in input $infile : $line\n"; }
+	if (@words != 3) { die "error in input $infile : $line\n"; }
 	push (@ins_list, $words[0]);
+	push (@dst_list, $words[1]);
+	push (@md5_list, $words[2]);
     }
     close (FILE);
 
-    return @ins_list;
+    return (\@ins_list, \@dst_list, \@md5_list);
 }
 
Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.pt
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/rawfix.pt	(revision 42400)
+++ /trunk/tools/eam/rawfix.20230221/src/rawfix.pt	(revision 42400)
@@ -0,0 +1,305 @@
+## pantasks scripts to manage the raw exposure data repair / cleanup
+
+#### the scripts below take the input files (YYYYMMDD/*.inst.txt) 
+#### and run get_md5s_instances.pl on the hosts associated with those lists
+#### the outputs are files (YYYYMMDD/*.md5s.txt) 
+
+$TPOLL = 5
+$TEXEC = 5
+$RPOLL = 5
+$REXEC = 5
+$VERBOSE = 1
+$NETWORK = 1
+
+# use this to find the files in this local directory
+$mypath = `pwd`
+
+if (not($?RAWFIX_INIT))
+  book init md5sum.book
+  book init md5host.book
+  $RAWFIX_INIT = 1
+end
+
+macro load.hosts
+  # read the list of possible hosts from the file
+  # for each host, create a variable $host:name = 0
+  # when the host is running, set this to 1
+
+  list allhosts -x "cat allhosts.txt"
+
+  for i 0 $allhosts:n
+    $name = $allhosts:$i
+    $host:$name = 0
+  end
+end
+
+# select nights which are ready for md5sum calcs
+# output: md5sum.book
+task nights.md5sum.load
+  host local
+  periods -poll $TPOLL
+  periods -exec $TEXEC
+  periods -timeout 30
+
+  # only run one of these tasks at a time
+  npending 1
+
+  task.exec
+    command check.rawfix.sh state md5sum.new 
+  end
+
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum.book
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# output: md5host.book
+task nights.md5sum.loadhosts
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+  npending 1
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages md5sum.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new nights in md5sum.book (pantaskState == INIT)
+    book getpage md5sum.book 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword md5sum.book $pageName pantaskState RUN
+    book getword md5sum.book $pageName dateobs -var DATEOBS
+    option $pageName
+    # save the page so we can use it below
+
+    command get_hosts_md5s.sh $DATEOBS
+  end
+
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    queuesize stdout -var Nhosts
+    queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host.book
+    book setword md5sum.book $options:0 NHOST {$Nhosts - 1}
+    book setword md5sum.book $options:0 NDONE 0
+    # use the queuesize (Nhosts) to track how many jobs are done / to be done
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+    book setword md5sum.book $options:0 pantasksQuality FAIL
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword md5sum.book $options:0 pantasksQuality CRASH
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword md5sum.book $options:0 pantasksQuality TIMEOUT
+  end
+end
+
+# output: md5host.book
+task nights.md5sum.runhosts
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages md5host.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new entries in md5host.book (pantaskState == INIT)
+    book getpage md5host.book 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword md5host.book $pageName pantaskState RUN
+    book getword md5host.book $pageName dateobs -var DATEOBS
+    book getword md5host.book $pageName md5host -var MD5HOST
+
+    option $pageName $DATEOBS
+    # save DATEOBS so we can find the md5sum.book entry
+
+    # XXX test: use test host
+    $TESTHOST = ipp060
+    host -required $TESTHOST
+    if (not($?host:$TESTHOST)) 
+      echo "host not defined, creating : $TESTHOST"
+      $host:$TESTHOST = 0
+    end
+    if (not($host:$TESTHOST)) 
+      control host add $TESTHOST
+      $host:$TESTHOST = 1
+    end
+    # host -required $MD5HOST 
+
+    ## check if this host needs to be launched
+
+    $input = $mypath/$DATEOBS/$MD5HOST.inst.txt
+
+    # need to make sure this will work on the ipp b-nodes:
+    # command $mypath/get_md5s_instances.pl --hostname $MD5HOST --input $input
+    command $mypath/get_md5s_instances.pl --hostname $TESTHOST --input $input
+  end
+
+  task.exit 0
+    nights.mdsum.exit OK
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    nights.mdsum.exit FAIL
+    showcommand failure
+  end
+
+  task.exit    crash
+    nights.mdsum.exit CRASH
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    nights.mdsum.exit TIMEOUT
+    showcommand timeout
+  end
+end
+
+# output: md5host.book
+task nights.md5sum.clean
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+  npending 1
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages md5sum.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for completed nights in md5sum.book (pantaskState == DONE)
+    book getpage md5sum.book 0 -var pageName -key pantaskState DONE
+    echo "checking this page: $pageName"
+    if ("$pageName" == "NULL") break
+
+    echo "cleaning this page: $pageName"
+    book listpage md5sum.book $pageName
+
+    # book setword md5sum.book $pageName pantaskState RUN
+    book getword md5sum.book $pageName dateobs -var DATEOBS
+    book getword md5sum.book $pageName pantasksQuality -var QUALITY
+    echo "cleaning $pageName, quality: $QUALITY"
+
+    option $pageName
+    # save the page so we can use it below
+
+    if ("$QUALITY" == "OK")
+       
+      command update.rawfix.sh $DATEOBS md5sum.done 
+    else
+      command update.rawfix.sh $DATEOBS md5sum.fail 
+    end
+  end
+
+  task.exit 0
+    # now that the entry is set to done in the db, remove the pantasks book entry
+    echo "cleaning this page: $options:0"
+    book delpage md5sum.book $options:0
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+    book setword md5sum.book $options:0 pantasksQuality FAIL
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword md5sum.book $options:0 pantasksQuality CRASH
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword md5sum.book $options:0 pantasksQuality TIMEOUT
+  end
+end
+
+######################### UTILITIES #################################
+
+macro nights.mdsum.exit
+  if ($0 != 2)
+    echo "USAGE: nights.mdsum.exit (quality)"
+    break
+  end
+
+  book setword md5host.book $options:0 pantaskState DONE
+  book getword md5sum.book $options:1 NHOST -var Nhosts
+  book getword md5sum.book $options:1 NDONE -var Ndone
+  $Ndone ++
+  book setword md5sum.book $options:1 NDONE $Ndone
+  if ("$1" != "OK")
+    book setword md5sum.book $options:1 pantasksQuality $1
+  end 
+  if ($Ndone == $Nhosts)
+    book setword md5sum.book $options:1 pantaskState DONE
+  end
+  book delpage md5host.book $options:0
+end
+
+macro showcommand -c "print out the current command (used within tasks)"
+  local n 
+
+  if ($VERBOSE)
+    $command = $taskarg:0
+    for n 1 $taskarg:n
+      $command = $command $taskarg:$n
+    end
+    date -var exitdate
+    if ($0 == 2)
+      echo ""
+      echo "$1 for: $command"
+      echo "job exit status: $JOB_STATUS"
+      echo "job host: $JOB_HOSTNAME"
+      echo "job dtime: $JOB_DTIME"
+      echo "job exit date: $exitdate"
+    else
+      echo "command: $command"
+    end    
+  end
+end
+
Index: /trunk/tools/eam/rawfix.20230221/src/update.bnodes.sh
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/update.bnodes.sh	(revision 42400)
+++ /trunk/tools/eam/rawfix.20230221/src/update.bnodes.sh	(revision 42400)
@@ -0,0 +1,22 @@
+#!/bin/csh
+
+if ($#argv != 2) then
+  echo "USAGE: update.bnodes.sh (push/pull) (test/commit)"
+  exit 2
+endif
+
+set options = "-auvn"
+if ($2 == "commit") set options = "-auv"
+
+if ($1 == "push") then
+  rsync $options ./ ippb07:rawfix.20230221/
+  exit 0
+endif
+
+if ($1 == "pull") then
+  rsync $options ippb07:rawfix.20230221/ ./
+  exit 0
+endif
+
+echo "ERROR: unknown mode $1"
+exit 2
Index: /trunk/tools/eam/rawfix.20230221/src/update.rawfix.sh
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/update.rawfix.sh	(revision 42399)
+++ /trunk/tools/eam/rawfix.20230221/src/update.rawfix.sh	(revision 42400)
@@ -2,5 +2,5 @@
 
 if ($#argv < 1) then
-  echo "USAGE: update.rawfix.sh (night) (state)"
+  echo "USAGE: update.rawfix.sh (night) (state) [quality]"
   echo "   EG: update.rawfix.sh 20100501 new"
   exit 2