Index: trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh
===================================================================
--- trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh	(revision 42406)
+++ trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh	(revision 42411)
@@ -61,15 +61,22 @@
   set mystate = $2
 
-  if ($mystate == "new") goto good_state
-  if ($mystate == "rsync") goto good_state
-  if ($mystate == "md5sum.new") goto good_state
-  if ($mystate == "md5sum.done") goto good_state
-  if ($mystate == "md5sum.fail") goto good_state
-  if ($mystate == "ckchip.new") goto good_state
-  if ($mystate == "ckchip.done") goto good_state
-  if ($mystate == "ckchip.fail") goto good_state
-  if ($mystate == "fixchip.new") goto good_state
-  if ($mystate == "fixchip.done") goto good_state
-  if ($mystate == "fixchip.fail") goto good_state
+  if ($mystate == "ckchip.new")    goto good_state
+  if ($mystate == "ckchip.done")   goto good_state
+  if ($mystate == "ckchip.fail")   goto good_state
+  if ($mystate == "fixchip.new")   goto good_state
+  if ($mystate == "fixchip.done")  goto good_state
+  if ($mystate == "fixchip.fail")  goto good_state
+  if ($mystate == "rsync_s1.new")  goto good_state
+  if ($mystate == "rsync_s1.done") goto good_state
+  if ($mystate == "rsync_s1.fail") goto good_state
+  if ($mystate == "md5sum.new")    goto good_state
+  if ($mystate == "md5sum.done")   goto good_state
+  if ($mystate == "md5sum.fail")   goto good_state
+  if ($mystate == "rsync_s2.new")  goto good_state
+  if ($mystate == "rsync_s2.done") goto good_state
+  if ($mystate == "rsync_s2.fail") goto good_state
+  if ($mystate == "md5chk.new")    goto good_state
+  if ($mystate == "md5chk.done")   goto good_state
+  if ($mystate == "md5chk.fail")   goto good_state
 
   echo "ERROR: unknown mode $mystate"
@@ -77,5 +84,6 @@
 
   good_state:
-  mysql $DBOPTS -e "select dateobs from nights where (state = '$mystate')"
+  set t = `date +%T..%N`
+  mysql $DBOPTS -e "select dateobs, curtime() as timestamp from nights where (state = '$mystate')" | awk -v T=$t '(NR == 1){print $0}(NR > 1){print $1, T}'
   exit 0
 endif
Index: trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl
===================================================================
--- trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl	(revision 42406)
+++ trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl	(revision 42411)
@@ -18,4 +18,6 @@
 my @instlist = glob ("$dateword/*.inst.txt");
 my @md5slist = glob ("$dateword/*.md5s.txt");
+
+my $Nfail = 0;
 
 foreach my $instfile (@instlist) {
@@ -39,5 +41,6 @@
 
     if ($md5match eq "NONE") {
-	print STDERR "missing md5sum file for $instfile\n";
+	print STDOUT "missing md5sum file for $instfile\n";
+	$Nfail ++;
 	next;
     }
@@ -57,5 +60,6 @@
     my $N2 = @instdata;
     if ($N1 != $N2) {
-	print STDERR "inconsistent number of lines in $md5match : $N1 vs $N2\n";
+	print STDOUT "inconsistent number of lines in $md5match : $N1 vs $N2\n";
+	$Nfail ++;
 	next;
     }
@@ -76,7 +80,16 @@
 
 	if ($md5tru ne $md5obs) {
-	    print STDERR "FAIL: $inst $state $md5tru $md5obs\n";
+	    print STDOUT "FAIL: $inst $state $md5tru $md5obs\n";
+	    $Nfail ++;
 	}
     }
+}
+
+if ($Nfail) { 
+    print STDOUT "ERROR: $Nfail errors\n";
+    exit 1;
+} else {
+    print STDOUT "SUCCESS: $Nfail errors\n";
+    exit 0;
 }
 
Index: trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl
===================================================================
--- trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl	(revision 42406)
+++ trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl	(revision 42411)
@@ -4,5 +4,4 @@
 my $time = time();
 
-use DBI;
 use Carp;
 use Pod::Usage qw( pod2usage );
Index: trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt
===================================================================
--- trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt	(revision 42406)
+++ trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt	(revision 42411)
@@ -30,8 +30,11 @@
   end
 
+  # need to delete DONE entries here so the book is not repopulated
+  # by already-running load tasks
   task.exit 0
     # convert the stdout 'queue' to entries in a book
     # fields in the table will become words in the page
     queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout ckchip.book
+    book delpage ckchip.book -key pantaskState DONE
   end
 
@@ -91,6 +94,6 @@
     # fields in the table will become words in the page
     # use the queuesize (Nhosts) to track how many jobs are done / to be done
+    book setword ckchip.book $options:0 pantaskState DONE
     exec update.rawfix.sh $options:0 ckchip.done
-    book delpage ckchip.book $options:0
   end
 
Index: trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt
===================================================================
--- trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt	(revision 42406)
+++ trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt	(revision 42411)
@@ -35,4 +35,5 @@
     # fields in the table will become words in the page
     queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip.book
+    book delpage fixchip.book -key pantaskState DONE
   end
 
@@ -91,4 +92,5 @@
     # fields in the table will become words in the page
     # use the queuesize (Nhosts) to track how many jobs are done / to be done
+    book setword fixchip.book $options:0 pantaskState DONE
     exec update.rawfix.sh $options:0 fixchip.done
     book delpage fixchip.book $options:0
Index: trunk/tools/eam/rawfix.20230221/src/rawfix.md5chk.pt
===================================================================
--- trunk/tools/eam/rawfix.20230221/src/rawfix.md5chk.pt	(revision 42411)
+++ trunk/tools/eam/rawfix.20230221/src/rawfix.md5chk.pt	(revision 42411)
@@ -0,0 +1,120 @@
+## pantasks scripts to manage the raw exposure data repair / cleanup
+## this is for the initial chip instance checks
+## use the top-level rawfix.pt to load
+
+# these tasks run the steps: 
+# -- check_md5s_dateobs.pl --dateobs YYYY/MM/DD
+
+if (not($?RAWFIX_MD5CHK_INIT))
+  book init md5chk.book
+  $RAWFIX_MD5CHK_INIT = 1
+end
+
+# select nights in state md5chk.new
+# output: md5chk.book
+task nights.md5chk.load
+  host local
+  periods -poll $TPOLL
+  periods -exec $TEXEC
+  periods -timeout 30
+
+  stdout $LOGDIR/md5chk.load.stdout.log  
+  stderr $LOGDIR/md5chk.load.stderr.log
+
+  # only run one of these tasks at a time
+  npending 1
+
+  task.exec
+    # command check.rawfix.sh state ckchip.done
+    command check.rawfix.sh state md5chk.new 
+  end
+
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5chk.book
+    book delpage md5chk.book -key pantaskState DONE
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task nights.md5chk.run
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+  npending 1
+
+  stdout $LOGDIR/md5chk.run.stdout.log  
+  stderr $LOGDIR/md5chk.run.stderr.log
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages md5chk.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new nights in md5chk.book (pantaskState == INIT)
+    book getpage md5chk.book 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword md5chk.book $pageName pantaskState RUN
+    book getword md5chk.book $pageName dateobs -var DATEOBS
+    option $pageName
+
+    substr $DATEOBS 0 4 YEAR
+    substr $DATEOBS 4 2 MONTH
+    substr $DATEOBS 6 2 DAY
+    sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
+
+    # send output to a file in the DATEOBS directory
+    stdout $DATEOBS/md5chk.log  
+
+    command check_md5s_dateobs.pl --dateobs $FULLDATE
+  end
+
+  ## XX delete the page from the book if done?
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    # use the queuesize (Nhosts) to track how many jobs are done / to be done
+    book setword md5chk.book $options:0 pantaskState DONE
+    exec update.rawfix.sh $options:0 md5chk.done
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+    book setword md5chk.book $options:0 pantasksQuality FAIL
+    exec update.rawfix.sh $options:0 md5chk.fail
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword md5chk.book $options:0 pantasksQuality CRASH
+    exec update.rawfix.sh $options:0 md5chk.fail
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword md5chk.book $options:0 pantasksQuality TIMEOUT
+    exec update.rawfix.sh $options:0 md5chk.fail
+  end
+end
+
Index: trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt
===================================================================
--- trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt	(revision 42406)
+++ trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt	(revision 42411)
@@ -35,4 +35,5 @@
     # fields in the table will become words in the page
     queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum.book
+    book delpage md5sum.book -key pantaskState DONE
   end
 
@@ -210,5 +211,5 @@
     
     # look for completed nights in md5sum.book (pantaskState == DONE)
-    book getpage md5sum.book 0 -var pageName -key pantaskState DONE
+    book getpage md5sum.book 0 -var pageName -key pantaskState STOP
     if ("$pageName" == "NULL") break
 
@@ -232,5 +233,5 @@
     # now that the entry is set to done in the db, remove the pantasks book entry
     echo "cleaning this page: $options:0"
-    book delpage md5sum.book $options:0
+    book setword md5sum.book $options:0 pantaskState DONE
   end
 
@@ -274,5 +275,5 @@
   end 
   if ($Ndone == $Nhosts)
-    book setword md5sum.book $options:1 pantaskState DONE
+    book setword md5sum.book $options:1 pantaskState STOP
   end
   book delpage md5host.book $options:0
Index: trunk/tools/eam/rawfix.20230221/src/rawfix.pt
===================================================================
--- trunk/tools/eam/rawfix.20230221/src/rawfix.pt	(revision 42406)
+++ trunk/tools/eam/rawfix.20230221/src/rawfix.pt	(revision 42411)
@@ -16,6 +16,13 @@
 end
 
+# these are listed in stage order, but the 
+# order is this file is not determinative, only the
+# change of state from STAGE1.new -> STAGE1.done -> STAGE2.new
 input rawfix.ckchip.pt
-# input rawfix.md5sum.pt
+input rawfix.fixchip.pt
+input rawfix.rsync_s1.pt
+input rawfix.md5sum.pt
+input rawfix.rsync_s2.pt
+input rawfix.md5chk.pt
 
 ######################### UTILITIES #################################
Index: trunk/tools/eam/rawfix.20230221/src/rawfix.rsync_s1.pt
===================================================================
--- trunk/tools/eam/rawfix.20230221/src/rawfix.rsync_s1.pt	(revision 42411)
+++ trunk/tools/eam/rawfix.20230221/src/rawfix.rsync_s1.pt	(revision 42411)
@@ -0,0 +1,126 @@
+## pantasks scripts to manage the raw exposure data repair / cleanup
+## this is for the initial chip instance checks
+## use the top-level rawfix.pt to load
+
+# these tasks run the steps: 
+# -- rsync -auv $SRCDIR/ $TGTDIR/
+
+$SRCDIR = /data/ipp060.0/eugene/rawfix.20230221
+$TGTDIR = /data/ippb07.0/home/eugene/rawfix.20230221
+
+if (not($?RAWFIX_RSYNC_S1_INIT))
+  book init rsync_s1.book
+  $RAWFIX_RSYNC_S1_INIT = 1
+end
+
+# select nights in state rsync_s1.new
+# output: rsync_s1.book
+task nights.rsync_s1.load
+  host local
+  periods -poll $TPOLL
+  periods -exec $TEXEC
+  periods -timeout 30
+
+  stdout $LOGDIR/rsync_s1.load.stdout.log  
+  stderr $LOGDIR/rsync_s1.load.stderr.log
+
+  # only run one of these tasks at a time
+  npending 1
+
+  task.exec
+    # command check.rawfix.sh state ckchip.done
+    command check.rawfix.sh state rsync_s1.new 
+  end
+
+  # need to delete DONE entries here so the book is not repopulated
+  # by already-running load tasks
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout rsync_s1.book
+    book delpage rsync_s1.book -key pantaskState DONE
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+###########################################################3333333
+
+task nights.rsync_s1.run
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+  npending 1
+
+  stdout $LOGDIR/rsync_s1.run.stdout.log  
+  stderr $LOGDIR/rsync_s1.run.stderr.log
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages rsync_s1.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new nights in rsync_s1.book (pantaskState == INIT)
+    book getpage rsync_s1.book 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    # echo "got a valid entry: $pageName"
+    # book listpage rsync_s1.book $pageName
+
+    book setword rsync_s1.book $pageName pantaskState RUN
+    book getword rsync_s1.book $pageName dateobs -var DATEOBS
+    option $pageName
+
+    # send output to a file in the DATEOBS directory
+    stdout $DATEOBS/rsync_s1.log  
+    stderr $DATEOBS/rsync_s1.log  
+
+    command rsync -auv $SRCDIR/$DATEOBS/ $TGTDIR/$DATEOBS/
+  end
+
+  ## do not delete the page from the book if done here
+  ## deletion has to happen after the .load
+  task.exit 0
+    echo "--- stdout : nights.rsync_s1.run ---"
+    queueprint stdout
+    exec update.rawfix.sh $options:0 rsync_s1.done
+    book setword rsync_s1.book $options:0 pantaskState DONE
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+    book setword rsync_s1.book $options:0 pantasksQuality FAIL
+    exec update.rawfix.sh $options:0 rsync_s1.fail
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword rsync_s1.book $options:0 pantasksQuality CRASH
+    exec update.rawfix.sh $options:0 rsync_s1.fail
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword rsync_s1.book $options:0 pantasksQuality TIMEOUT
+    exec update.rawfix.sh $options:0 rsync_s1.fail
+  end
+end
+
Index: trunk/tools/eam/rawfix.20230221/src/rawfix.rsync_s2.pt
===================================================================
--- trunk/tools/eam/rawfix.20230221/src/rawfix.rsync_s2.pt	(revision 42411)
+++ trunk/tools/eam/rawfix.20230221/src/rawfix.rsync_s2.pt	(revision 42411)
@@ -0,0 +1,124 @@
+## pantasks scripts to manage the raw exposure data repair / cleanup
+## this is for the initial chip instance checks
+## use the top-level rawfix.pt to load
+
+# these tasks run the steps: 
+# -- rsync -auv $SRCDIR/ $TGTDIR/
+
+$SRCDIR = /data/ipp060.0/eugene/rawfix.20230221
+$TGTDIR = /data/ippb07.0/home/eugene/rawfix.20230221
+
+if (not($?RAWFIX_RSYNC_S2_INIT))
+  book init rsync_s2.book
+  $RAWFIX_RSYNC_S2_INIT = 1
+end
+
+# select nights in state rsync_s2.new
+# output: rsync_s2.book
+task nights.rsync_s2.load
+  host local
+  periods -poll $TPOLL
+  periods -exec $TEXEC
+  periods -timeout 30
+
+  stdout $LOGDIR/rsync_s2.load.stdout.log  
+  stderr $LOGDIR/rsync_s2.load.stderr.log
+
+  # only run one of these tasks at a time
+  npending 1
+
+  task.exec
+    # command check.rawfix.sh state ckchip.done
+    command check.rawfix.sh state rsync_s2.new 
+  end
+
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout rsync_s2.book
+    book delpage rsync_s2.book -key pantaskState DONE
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+###########################################################3333333
+
+task nights.rsync_s2.run
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+  npending 1
+
+  stdout $LOGDIR/rsync_s2.run.stdout.log  
+  stderr $LOGDIR/rsync_s2.run.stderr.log
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages rsync_s2.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new nights in rsync_s2.book (pantaskState == INIT)
+    book getpage rsync_s2.book 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    # echo "got a valid entry: $pageName"
+    # book listpage rsync_s2.book $pageName
+
+    book setword rsync_s2.book $pageName pantaskState RUN
+    book getword rsync_s2.book $pageName dateobs -var DATEOBS
+    option $pageName
+
+    # send output to a file in the DATEOBS directory
+    stdout $DATEOBS/rsync_s2.log  
+    stderr $DATEOBS/rsync_s2.log  
+
+    command rsync -auv $TGTDIR/$DATEOBS/ $SRCDIR/$DATEOBS/
+  end
+
+  ## do not delete the page from the book if done here
+  ## deletion has to happen after the .load
+  task.exit 0
+    echo "--- stdout : nights.rsync_s2.run ---"
+    queueprint stdout
+    exec update.rawfix.sh $options:0 rsync_s2.done
+    book setword rsync_s2.book $options:0 pantaskState DONE
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+    book setword rsync_s2.book $options:0 pantasksQuality FAIL
+    exec update.rawfix.sh $options:0 rsync_s2.fail
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword rsync_s2.book $options:0 pantasksQuality CRASH
+    exec update.rawfix.sh $options:0 rsync_s2.fail
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword rsync_s2.book $options:0 pantasksQuality TIMEOUT
+    exec update.rawfix.sh $options:0 rsync_s2.fail
+  end
+end
+