Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt	(revision 42405)
+++ /trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt	(revision 42405)
@@ -0,0 +1,115 @@
+## pantasks scripts to manage the raw exposure data repair / cleanup
+## this is for the initial chip instance checks
+## use the top-level rawfix.pt to load
+
+# these tasks run the steps: 
+# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt
+# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD)
+
+if (not($?RAWFIX_CKCHIP_INIT))
+  book init ckchip.book
+  $RAWFIX_CKCHIP_INIT = 1
+end
+
+# select nights in state ckchip.new
+# output: ckchip.book
+task nights.ckchip.load
+  host local
+  periods -poll $TPOLL
+  periods -exec $TEXEC
+  periods -timeout 30
+
+  stdout $LOGDIR/ckchip.load.stdout.log  
+  stderr $LOGDIR/ckchip.load.stderr.log
+
+  # only run one of these tasks at a time
+  npending 1
+
+  task.exec
+    command check.rawfix.sh state ckchip.new 
+  end
+
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout ckchip.book
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task nights.ckchip.run
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+  npending 1
+
+  stdout $LOGDIR/ckchip.run.stdout.log  
+  stderr $LOGDIR/ckchip.run.stderr.log
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages ckchip.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new nights in ckchip.book (pantaskState == INIT)
+    book getpage ckchip.book 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword ckchip.book $pageName pantaskState RUN
+    book getword ckchip.book $pageName dateobs -var DATEOBS
+    option $pageName
+
+    substr $DATEOBS 0 4 YEAR
+    substr $DATEOBS 4 2 MONTH
+    substr $DATEOBS 6 2 DAY
+
+    sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
+
+    command check_chip_locations.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt
+  end
+
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    queuesize stdout -var Nhosts
+    queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host.book
+    book setword ckchip.book $options:0 NHOST {$Nhosts - 1}
+    book setword ckchip.book $options:0 NDONE 0
+    # use the queuesize (Nhosts) to track how many jobs are done / to be done
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+    book setword ckchip.book $options:0 pantasksQuality FAIL
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword ckchip.book $options:0 pantasksQuality CRASH
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword ckchip.book $options:0 pantasksQuality TIMEOUT
+  end
+end
+
Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt	(revision 42405)
+++ /trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt	(revision 42405)
@@ -0,0 +1,280 @@
+## pantasks scripts to manage the raw exposure data repair / cleanup
+## this is for the md5sum calculations
+## use the top-level rawfix.pt to load
+
+#### the scripts below take the input files (YYYYMMDD/*.inst.txt) 
+#### and run get_md5s_instances.pl on the hosts associated with those lists
+#### the outputs are files (YYYYMMDD/*.md5s.txt) 
+
+if (not($?RAWFIX_MD5SUM_INIT))
+  book init md5sum.book
+  book init md5host.book
+  $RAWFIX_MD5SUM_INIT = 1
+end
+
+# select nights which are ready for md5sum calcs
+# output: md5sum.book
+task nights.md5sum.load
+  host local
+  periods -poll $TPOLL
+  periods -exec $TEXEC
+  periods -timeout 30
+
+  stdout $LOGDIR/md5sum.load.stdout.log  
+  stderr $LOGDIR/md5sum.load.stderr.log
+
+  # only run one of these tasks at a time
+  npending 1
+
+  task.exec
+    command check.rawfix.sh state md5sum.new 
+  end
+
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum.book
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# output: md5host.book
+task nights.md5sum.loadhosts
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+  npending 1
+
+  stdout $LOGDIR/md5sum.loadhosts.stdout.log  
+  stderr $LOGDIR/md5sum.loadhosts.stderr.log
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages md5sum.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new nights in md5sum.book (pantaskState == INIT)
+    book getpage md5sum.book 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword md5sum.book $pageName pantaskState RUN
+    book getword md5sum.book $pageName dateobs -var DATEOBS
+    option $pageName
+    # save the page so we can use it below
+
+    # write an entry point in the log for this night
+    sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $DATEOBS
+    date -var mydate
+    exec echo "# start runhosts $mydate" >> $tmpline
+
+    command get_hosts_md5s.sh $DATEOBS
+  end
+
+  task.exit 0
+    # convert the stdout 'queue' to entries in a book
+    # fields in the table will become words in the page
+    queuesize stdout -var Nhosts
+    queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host.book
+    book setword md5sum.book $options:0 NHOST {$Nhosts - 1}
+    book setword md5sum.book $options:0 NDONE 0
+    # use the queuesize (Nhosts) to track how many jobs are done / to be done
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+    book setword md5sum.book $options:0 pantasksQuality FAIL
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword md5sum.book $options:0 pantasksQuality CRASH
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword md5sum.book $options:0 pantasksQuality TIMEOUT
+  end
+end
+
+# output: md5host.book
+task nights.md5sum.runhosts
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+
+  stdout $LOGDIR/md5sum.runhosts.stdout.log  
+  stderr $LOGDIR/md5sum.runhosts.stderr.log
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages md5host.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new entries in md5host.book (pantaskState == INIT)
+    book getpage md5host.book 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword md5host.book $pageName pantaskState RUN
+    book getword md5host.book $pageName dateobs -var DATEOBS
+    book getword md5host.book $pageName md5host -var MD5HOST
+
+    option $pageName $DATEOBS
+    # save DATEOBS so we can find the md5sum.book entry
+
+    # XXX TEST: $USE_HOST = ipp060
+
+    $USE_HOST = $MD5HOST
+    host -required $USE_HOST
+    ## check if this host needs to be launched
+    if (not($?host:$USE_HOST)) 
+      echo "host not defined, creating : $USE_HOST"
+      $host:$USE_HOST = 0
+    end
+    if (not($host:$USE_HOST)) 
+      control host add $USE_HOST
+      $host:$USE_HOST = 1
+    end
+
+    ## even when testing, use the original hostname here:
+    $input = $mypath/$DATEOBS/$MD5HOST.inst.txt
+
+    # need to make sure this will work on the ipp b-nodes:
+    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
+    periods -exec 0.05
+    command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input
+  end
+
+  task.exit 0
+    nights.mdsum.exit OK
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    nights.mdsum.exit FAIL
+    showcommand failure
+  end
+
+  task.exit    crash
+    nights.mdsum.exit CRASH
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    nights.mdsum.exit TIMEOUT
+    showcommand timeout
+  end
+end
+
+# output: md5host.book
+task nights.md5sum.clean
+
+  periods      -poll $RPOLL
+  periods      -exec $REXEC
+  periods      -timeout 60
+  npending 1
+
+  stdout $LOGDIR/md5sum.clean.stdout.log  
+  stderr $LOGDIR/md5sum.clean.stderr.log
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $REXEC
+
+    book npages md5sum.book -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for completed nights in md5sum.book (pantaskState == DONE)
+    book getpage md5sum.book 0 -var pageName -key pantaskState DONE
+    if ("$pageName" == "NULL") break
+
+    # book listpage md5sum.book $pageName
+
+    # book setword md5sum.book $pageName pantaskState RUN
+    book getword md5sum.book $pageName dateobs -var DATEOBS
+    book getword md5sum.book $pageName pantasksQuality -var QUALITY
+
+    option $pageName
+    # save the page so we can use it below
+
+    if ("$QUALITY" == "OK")
+      command update.rawfix.sh $DATEOBS md5sum.done 
+    else
+      command update.rawfix.sh $DATEOBS md5sum.fail 
+    end
+  end
+
+  task.exit 0
+    # now that the entry is set to done in the db, remove the pantasks book entry
+    echo "cleaning this page: $options:0"
+    book delpage md5sum.book $options:0
+  end
+
+  # exit values other than 0:
+  task.exit    default
+    showcommand failure
+    book setword md5sum.book $options:0 pantasksQuality FAIL
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword md5sum.book $options:0 pantasksQuality CRASH
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword md5sum.book $options:0 pantasksQuality TIMEOUT
+  end
+end
+
+######################### UTILITIES #################################
+
+macro nights.mdsum.exit
+  if ($0 != 2)
+    echo "USAGE: nights.mdsum.exit (quality)"
+    break
+  end
+
+  book setword md5host.book $options:0 pantaskState DONE
+  book getword md5sum.book $options:1 NHOST -var Nhosts
+  book getword md5sum.book $options:1 NDONE -var Ndone
+  $Ndone ++
+  book setword md5sum.book $options:1 NDONE $Ndone
+  if ("$1" != "OK")
+    book setword md5sum.book $options:1 pantasksQuality $1
+    sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $options:1
+    exec echo "failure $1 $options:0" >> $tmpline
+  else
+    exec echo "success $1 $options:0" >> $tmpline
+  end 
+  if ($Ndone == $Nhosts)
+    book setword md5sum.book $options:1 pantaskState DONE
+  end
+  book delpage md5host.book $options:0
+end
+
Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.pt
===================================================================
--- /trunk/tools/eam/rawfix.20230221/src/rawfix.pt	(revision 42404)
+++ /trunk/tools/eam/rawfix.20230221/src/rawfix.pt	(revision 42405)
@@ -20,275 +20,8 @@
 end
 
-if (not($?RAWFIX_INIT))
-  book init md5sum.book
-  book init md5host.book
-  $RAWFIX_INIT = 1
-end
-
-# select nights which are ready for md5sum calcs
-# output: md5sum.book
-task nights.md5sum.load
-  host local
-  periods -poll $TPOLL
-  periods -exec $TEXEC
-  periods -timeout 30
-
-  stdout $LOGDIR/md5sum.load.stdout.log  
-  stderr $LOGDIR/md5sum.load.stderr.log
-
-  # only run one of these tasks at a time
-  npending 1
-
-  task.exec
-    command check.rawfix.sh state md5sum.new 
-  end
-
-  task.exit 0
-    # convert the stdout 'queue' to entries in a book
-    # fields in the table will become words in the page
-    queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum.book
-  end
-
-  # exit values other than 0:
-  task.exit    default
-    showcommand failure
-  end
-
-  task.exit    crash
-    showcommand crash
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-
-# output: md5host.book
-task nights.md5sum.loadhosts
-
-  periods      -poll $RPOLL
-  periods      -exec $REXEC
-  periods      -timeout 60
-  npending 1
-
-  stdout $LOGDIR/md5sum.loadhosts.stdout.log  
-  stderr $LOGDIR/md5sum.loadhosts.stderr.log
-
-  task.exec
-    # if we are unable to run the 'exec', use a long retry time
-    periods -exec $REXEC
-
-    book npages md5sum.book -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    
-    # look for new nights in md5sum.book (pantaskState == INIT)
-    book getpage md5sum.book 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-
-    book setword md5sum.book $pageName pantaskState RUN
-    book getword md5sum.book $pageName dateobs -var DATEOBS
-    option $pageName
-    # save the page so we can use it below
-
-    # write an entry point in the log for this night
-    sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $DATEOBS
-    date -var mydate
-    exec echo "# start runhosts $mydate" >> $tmpline
-
-    command get_hosts_md5s.sh $DATEOBS
-  end
-
-  task.exit 0
-    # convert the stdout 'queue' to entries in a book
-    # fields in the table will become words in the page
-    queuesize stdout -var Nhosts
-    queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host.book
-    book setword md5sum.book $options:0 NHOST {$Nhosts - 1}
-    book setword md5sum.book $options:0 NDONE 0
-    # use the queuesize (Nhosts) to track how many jobs are done / to be done
-  end
-
-  # exit values other than 0:
-  task.exit    default
-    showcommand failure
-    book setword md5sum.book $options:0 pantasksQuality FAIL
-  end
-
-  task.exit    crash
-    showcommand crash
-    book setword md5sum.book $options:0 pantasksQuality CRASH
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-    book setword md5sum.book $options:0 pantasksQuality TIMEOUT
-  end
-end
-
-# output: md5host.book
-task nights.md5sum.runhosts
-
-  periods      -poll $RPOLL
-  periods      -exec $REXEC
-  periods      -timeout 60
-
-  stdout $LOGDIR/md5sum.runhosts.stdout.log  
-  stderr $LOGDIR/md5sum.runhosts.stderr.log
-
-  task.exec
-    # if we are unable to run the 'exec', use a long retry time
-    periods -exec $REXEC
-
-    book npages md5host.book -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    
-    # look for new entries in md5host.book (pantaskState == INIT)
-    book getpage md5host.book 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-
-    book setword md5host.book $pageName pantaskState RUN
-    book getword md5host.book $pageName dateobs -var DATEOBS
-    book getword md5host.book $pageName md5host -var MD5HOST
-
-    option $pageName $DATEOBS
-    # save DATEOBS so we can find the md5sum.book entry
-
-    # XXX TEST: $USE_HOST = ipp060
-
-    $USE_HOST = $MD5HOST
-    host -required $USE_HOST
-    ## check if this host needs to be launched
-    if (not($?host:$USE_HOST)) 
-      echo "host not defined, creating : $USE_HOST"
-      $host:$USE_HOST = 0
-    end
-    if (not($host:$USE_HOST)) 
-      control host add $USE_HOST
-      $host:$USE_HOST = 1
-    end
-
-    ## even when testing, use the original hostname here:
-    $input = $mypath/$DATEOBS/$MD5HOST.inst.txt
-
-    # need to make sure this will work on the ipp b-nodes:
-    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
-    periods -exec 0.05
-    command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input
-  end
-
-  task.exit 0
-    nights.mdsum.exit OK
-  end
-
-  # exit values other than 0:
-  task.exit    default
-    nights.mdsum.exit FAIL
-    showcommand failure
-  end
-
-  task.exit    crash
-    nights.mdsum.exit CRASH
-    showcommand crash
-  end
-
-  # operation times out?
-  task.exit    timeout
-    nights.mdsum.exit TIMEOUT
-    showcommand timeout
-  end
-end
-
-# output: md5host.book
-task nights.md5sum.clean
-
-  periods      -poll $RPOLL
-  periods      -exec $REXEC
-  periods      -timeout 60
-  npending 1
-
-  stdout $LOGDIR/md5sum.clean.stdout.log  
-  stderr $LOGDIR/md5sum.clean.stderr.log
-
-  task.exec
-    # if we are unable to run the 'exec', use a long retry time
-    periods -exec $REXEC
-
-    book npages md5sum.book -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    
-    # look for completed nights in md5sum.book (pantaskState == DONE)
-    book getpage md5sum.book 0 -var pageName -key pantaskState DONE
-    if ("$pageName" == "NULL") break
-
-    # book listpage md5sum.book $pageName
-
-    # book setword md5sum.book $pageName pantaskState RUN
-    book getword md5sum.book $pageName dateobs -var DATEOBS
-    book getword md5sum.book $pageName pantasksQuality -var QUALITY
-
-    option $pageName
-    # save the page so we can use it below
-
-    if ("$QUALITY" == "OK")
-      command update.rawfix.sh $DATEOBS md5sum.done 
-    else
-      command update.rawfix.sh $DATEOBS md5sum.fail 
-    end
-  end
-
-  task.exit 0
-    # now that the entry is set to done in the db, remove the pantasks book entry
-    echo "cleaning this page: $options:0"
-    book delpage md5sum.book $options:0
-  end
-
-  # exit values other than 0:
-  task.exit    default
-    showcommand failure
-    book setword md5sum.book $options:0 pantasksQuality FAIL
-  end
-
-  task.exit    crash
-    showcommand crash
-    book setword md5sum.book $options:0 pantasksQuality CRASH
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-    book setword md5sum.book $options:0 pantasksQuality TIMEOUT
-  end
-end
+input rawfix.setup.pt
+input rawfix.md5sum.pt
 
 ######################### UTILITIES #################################
-
-macro nights.mdsum.exit
-  if ($0 != 2)
-    echo "USAGE: nights.mdsum.exit (quality)"
-    break
-  end
-
-  book setword md5host.book $options:0 pantaskState DONE
-  book getword md5sum.book $options:1 NHOST -var Nhosts
-  book getword md5sum.book $options:1 NDONE -var Ndone
-  $Ndone ++
-  book setword md5sum.book $options:1 NDONE $Ndone
-  if ("$1" != "OK")
-    book setword md5sum.book $options:1 pantasksQuality $1
-    sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $options:1
-    exec echo "failure $1 $options:0" >> $tmpline
-  else
-    exec echo "success $1 $options:0" >> $tmpline
-  end 
-  if ($Ndone == $Nhosts)
-    book setword md5sum.book $options:1 pantaskState DONE
-  end
-  book delpage md5host.book $options:0
-end
 
 macro showcommand -c "print out the current command (used within tasks)"