Index: /tags/ipp-20170121/tools/check_system.sh
===================================================================
--- /tags/ipp-20170121/tools/check_system.sh	(revision 40100)
+++ /tags/ipp-20170121/tools/check_system.sh	(revision 40100)
@@ -0,0 +1,321 @@
+#!/bin/csh -f
+
+if ( ($#argv != 1) && ($#argv != 2) ) then
+  echo "USAGE: check_system.sh (mode)"
+  echo "  modes: start.server, setup, run, up, status, tasks, stop, shutdown, controller_status"
+  echo "  modes: hoston <host>, hostoff <host> (Add/Remove a host _once_ to/from pantasks), hostcheck <host> (Check host in pantasks)"
+  echo "  modes: reap (set controller to reap jobs not spawn new ones)"
+  echo "  modes: fullrun (set controller to normal run state)"
+  echo "  modes: jobstack (state) (have controller report jobs in given state)"
+  echo "  modes: listdirs (list currently active directories)"
+  echo "  modes: stop.block.restart"
+  echo
+  exit 2
+endif
+
+# "dirs" contains the current complete list
+# other directories currently skipped:
+# ~ippmops/stdscience_ws detrend addstar deepstack replication
+# previously, we split out update, publishing, 
+
+# set dirs = ( cleanup distribution registration stack stdscience summitcopy pstamp ~ippqub/stdscience_ws )
+## active list as of 20170316 MEH
+set dirs = ( registration stdscience summitcopy pstamp stack cleanup distribution )
+## stack being used for w-stack MEH 20170714 -- finished for now
+#set dirs = ( registration stdscience summitcopy pstamp  cleanup distribution )
+
+if ("$1" == "start.server") then
+  foreach dir ( $dirs )
+    set server = `grep "PANTASKS_SERVER " $dir/ptolemy.rc | awk '{print $2}'`
+    if ($status) then
+      echo "PANTASKS_SERVER is not defined for $dir"
+      continue
+    endif
+
+    # check for alternate user home directory
+    set userdir = `dirname $dir`
+
+    # if a pantasks directory is owned by another user, ssh to the server as that user
+    # NOTE: with this code, the pantasks directory must be a subdirectory of the user's home directory
+    set myUser = `whoami`
+    if ($userdir != ".") then
+      set myUser = `basename $userdir`
+      if (! -d ~$myUser) then
+        echo "failed to determine ops user for $dir, exiting"
+        exit 2
+      endif
+    endif
+
+    ssh $myUser@$server start_server.sh $dir
+  end
+  exit 0
+endif
+
+if ("$1" == "start") then
+  echo "NOTE: the option 'start' has been replaced with 'setup'"
+  exit 1
+endif
+
+if ($1 == "listdirs") then
+    echo "-- complete list of currently active pantasks directories --" 
+    foreach dir ($dirs)
+        echo "$dir"
+    end
+    exit 0
+endif
+
+if ($1 == "hostoff") then
+    foreach dir ($dirs)
+	echo "Removing $2 from $dir"
+	echo "controller host off $2" | pantasks_client -c $dir/ptolemy.rc >& /dev/null
+    end
+    exit 0
+endif
+
+if ($1 == "hoston") then
+    foreach dir ($dirs)
+	echo "Adding $2 to $dir"
+	echo "controller host on $2" | pantasks_client -c $dir/ptolemy.rc >& /dev/null
+    end
+    exit 0
+endif
+
+if ($1 == "hostcheck") then
+    foreach dir ($dirs)
+        echo "Checking $2 in $dir"
+        echo "controller host check $2" | pantasks_client -c $dir/ptolemy.rc | grep "host $2"
+    end
+    exit 0
+endif
+
+if ($1 == "reap") then
+    foreach dir ($dirs)
+	echo "setting $dir to 'reap'"
+	echo "controller run reap" | pantasks_client -c $dir/ptolemy.rc >& /dev/null
+    end
+    exit 0
+endif
+
+if ($1 == "fullrun") then
+    foreach dir ($dirs)
+	echo "setting $dir to 'run all'"
+	echo "controller run all" | pantasks_client -c $dir/ptolemy.rc >& /dev/null
+    end
+    exit 0
+endif
+
+if ($1 == "jobstack") then
+    if ($#argv != 2) then
+      echo "USAGE: check_system.sh jobstack (state)"
+      exit 2
+    endif
+    foreach dir ($dirs)
+	echo "jobs in state $2 for $dir"
+	echo "controller jobstack $2" | pantasks_client -c $dir/ptolemy.rc
+    end
+    exit 0
+endif
+
+if ("$1" == "setup") then
+  foreach dir ( $dirs )
+    set server = `grep "PANTASKS_SERVER " $dir/ptolemy.rc | awk '{print $2}'`
+    if ($status) then
+      echo "PANTASKS_SERVER is not defined for $dir"
+      continue
+    endif
+  
+    echo "server input input; setup; exit 0" | pantasks_client -c $dir/ptolemy.rc >& /dev/null
+    if ($status) then
+      echo "pantasks server $dir is NOT running (host: $server)"
+    else
+      echo "pantasks server $dir has been started (host: $server)"
+    endif
+  end
+  exit 0
+endif
+
+if ("$1" == "run") then
+
+  foreach dir ($dirs)
+    set server = `grep "PANTASKS_SERVER " $dir/ptolemy.rc | awk '{print $2}'`
+    if ($status) then
+      echo "PANTASKS_SERVER is not defined for $dir"
+      continue
+    endif
+  
+    echo "run; exit 0" | pantasks_client -c $dir/ptolemy.rc >& /dev/null
+    if ($status) then
+      echo "pantasks server $dir is NOT running (host: $server)"
+    else
+      echo "pantasks server $dir is running (host: $server)"
+    endif
+  end
+  exit 0
+endif
+
+if ("$1" == "status") then
+
+  foreach dir ( $dirs )
+    echo "############################################################"
+    echo "#"
+    echo "#"
+    echo "# $dir"
+    echo "#"
+    echo "#"
+    echo "############################################################"
+    set server = `grep "PANTASKS_SERVER " $dir/ptolemy.rc | awk '{print $2}'`
+    if ($status) then
+      echo "PANTASKS_SERVER is not defined for $dir"
+      continue
+    endif
+  
+    echo "status; exit 0" | pantasks_client -c $dir/ptolemy.rc
+    if ($status) then
+      echo "pantasks server $dir is NOT running (host: $server)"
+    else
+      echo "pantasks server $dir is active (host: $server)"
+    endif
+  end
+  exit 0
+endif
+
+if ("$1" == "tasks") then
+  foreach dir ( $dirs )
+    set server = `grep "PANTASKS_SERVER " $dir/ptolemy.rc | awk '{print $2}'`
+    if ($status) then
+      echo "PANTASKS_SERVER is not defined for $dir"
+      continue
+    endif
+    
+    echo "status -tasks; exit 0" | pantasks_client -c $dir/ptolemy.rc
+    if ($status) then
+      echo "pantasks server $dir is NOT running (host: $server)"
+    else
+      echo "pantasks server $dir is active (host: $server)"
+    endif
+  end
+  exit 0
+endif
+
+if ($1 == "controller_status") then
+    foreach dir ($dirs)
+        echo "Checking controller status in $dir"
+        echo "controller status; exit 0" | pantasks_client -c $dir/ptolemy.rc
+    end
+    exit 0
+endif
+
+if ("$1" == "up") then
+  foreach dir ( $dirs )
+    set server = `grep "PANTASKS_SERVER " $dir/ptolemy.rc | awk '{print $2}'`
+    if ($status) then
+      echo "PANTASKS_SERVER is not defined for $dir"
+      continue
+    endif
+    
+    echo "status; exit 0" | pantasks_client -c $dir/ptolemy.rc >& /dev/null
+    if ($status) then
+      echo "pantasks server $dir is NOT running (host: $server)"
+    else
+      echo "pantasks server $dir is active (host: $server)"
+    endif
+  end
+  exit 0
+endif
+
+if ("$1" == "stop") then
+  echo "addstar server NOT being stopped!"
+  foreach dir ($dirs)
+    set server = `grep "PANTASKS_SERVER " $dir/ptolemy.rc | awk '{print $2}'`
+    if ($status) then
+      echo "PANTASKS_SERVER is not defined for $dir"
+      continue
+    endif
+  
+    echo "stop; exit 0" | pantasks_client -c $dir/ptolemy.rc >& /dev/null
+    if ($status) then
+      echo "pantasks server $dir is NOT running (host: $server)"
+    else
+      echo "pantasks server $dir is being stopped (host: $server)"
+    endif
+  end
+  exit 0
+endif
+
+if ("$1" == "shutdown") then
+  echo "addstar server NOT being shutdown!"
+  foreach dir ($dirs)
+    set server = `grep "PANTASKS_SERVER " $dir/ptolemy.rc | awk '{print $2}'`
+    if ($status) then
+      echo "PANTASKS_SERVER is not defined for $dir"
+      continue
+    endif
+  
+    echo "shutdown now; exit 0" | pantasks_client -c $dir/ptolemy.rc >& /dev/null
+    if ($status) then
+      echo "pantasks server $dir is NOT running (host: $server)"
+    else
+      echo "pantasks server $dir is being shut down (host: $server)"
+    endif
+  end
+  exit 0
+endif
+
+if ("$1" == "stop.block.restart") then
+
+  # set all controllers to reap first, then block waiting for clear task list:
+  foreach dir ($dirs)
+    echo "setting $dir to 'reap'"
+    echo "controller run reap" | pantasks_client -c $dir/ptolemy.rc >& /dev/null
+  end
+
+  set tmpfile = `mktemp /tmp/pantasks.XXXXXX`
+
+  # block on clearing jobs from all pantasks:
+  foreach dir ($dirs)
+    echo "waiting for jobs to clear on $dir"
+
+    set Njobs = 1
+    set Nloop = 0
+    while ($Nloop < 30)
+      echo "control status -nohost -state busy" | pantasks_client -c $dir/ptolemy.rc >& $tmpfile
+      set Njobs = `awk '($3 == "BUSY"){print $0}' $tmpfile | wc -l`
+      if ($Njobs == 0) break
+      sleep 10
+    end
+    
+    if ($Njobs > 0) then
+      echo "WARNING : jobs are still running after 5 minutes: needs a manual check on $dir"
+      exit 1
+    endif
+  end
+  
+  # all pantasks are clear; now shutdown and restart:
+  ./check_system.sh shutdown
+  if ($status) then
+    echo "problem shutting down"
+    exit 2
+  endif
+
+  ./check_system.sh start.server
+  if ($status) then
+    echo "problem starting servers"
+    exit 2
+  endif
+
+  ./check_system.sh setup
+  if ($status) then
+    echo "problem setting up servers"
+    exit 2
+  endif
+
+  ./check_system.sh run
+  if ($status) then
+    echo "problem running servers"
+    exit 2
+  endif
+  exit 0
+endif
+
+echo "unknown mode $1"
+exit 1