Index: trunk/tools/eam/cluster.checks/check.dataeugene =================================================================== --- trunk/tools/eam/cluster.checks/check.dataeugene (revision 41456) +++ trunk/tools/eam/cluster.checks/check.dataeugene (revision 41456) @@ -0,0 +1,68 @@ +#!/usr/bin/env mana + +list hosts + ipp004 + ipp005 + ipp006 + ipp007 + ipp008 + ipp009 + ipp010 + ipp011 + ipp012 + ipp013 + ipp014 + ipp015 + ipp016 + ipp017 + ipp018 + ipp019 + ipp020 + ipp021 + ipp023 + ipp024 + ipp025 + ipp026 + ipp027 + ipp028 + ipp029 + ipp030 + ipp031 + ipp032 + ipp033 + ipp034 + ipp035 + ipp036 + ipp037 + ipp038 + ipp039 + ipp040 + ipp041 + ipp042 + ipp043 + ipp044 + ipp045 + ipp046 + ipp047 + ipp048 + ipp049 + ipp050 + ipp051 + ipp052 + ipp053 +end + +macro check.disks + break -auto off + exec rm -f data.eugene.dat + for i 0 $hosts:n + echo $hosts:$i + exec echo "--- $hosts:$i ---" >> data.eugene.dat + exec ls -l /data/$hosts:$i\.0/eugene >> data.eugene.dat + end +end + +if ($SCRIPT) + check.disks + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.df =================================================================== --- trunk/tools/eam/cluster.checks/check.df (revision 41456) +++ trunk/tools/eam/cluster.checks/check.df (revision 41456) @@ -0,0 +1,34 @@ +#!/usr/bin/env mana + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro remote.call + if ($0 != 2) + echo "USAGE: remote.call (host)" + break + end + $host = $1 + + echo host: $host + break -auto off + exec ssh $host df + break -auto on +end + +macro go + if ($0 != 1) + echo "USAGE: go" + break + end + + node.loop +end + +if ($SCRIPT) + if ($argv:n != 0) + echo "USAGE: check.umount" + exit 1 + end + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.dirs =================================================================== --- trunk/tools/eam/cluster.checks/check.dirs (revision 41456) +++ trunk/tools/eam/cluster.checks/check.dirs (revision 41456) @@ -0,0 +1,19 @@ +#!/usr/bin/env mana + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro remote.call + echo $name + break -auto off + exec ll -d /data/$name.0/yanxia + break -auto on +end + +macro go + node.loop +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.disks =================================================================== --- trunk/tools/eam/cluster.checks/check.disks (revision 41456) +++ trunk/tools/eam/cluster.checks/check.disks (revision 41456) @@ -0,0 +1,111 @@ +#!/usr/bin/env mana + +list hosts + ipp004 + ipp005 + ipp006 + ipp007 + ipp008 + ipp009 + ipp010 + ipp011 + ipp012 + ipp013 + ipp014 +# ipp015 + ipp016 + ipp017 + ipp018 + ipp019 + ipp020 + ipp021 + ipp023 + ipp024 + ipp025 + ipp026 + ipp027 + ipp028 + ipp029 + ipp030 + ipp031 + ipp032 + ipp033 + ipp034 + ipp035 + ipp036 + ipp037 + ipp038 + ipp039 + ipp040 + ipp041 + ipp042 + ipp043 + ipp044 + ipp045 + ipp046 + ipp047 + ipp048 + ipp049 + ipp050 + ipp051 + ipp052 + ipp053 + ipp054 + ipp055 + ipp056 + ipp057 + ipp058 + ipp059 + ipp060 + ipp061 + ipp062 + ipp063 + ipp064 + ipp065 + ipp066 +# ippc00 +# ippc01 +# ippc02 +# ippc03 +# ippc04 +# ippc05 +# ippc06 +# ippc07 +# ippc08 +# ippc09 +# ippc10 +# ippc11 +# ippc12 +# ippc13 +# ippc14 +# ippc15 +# ippc16 +# ippc17 +# ippc18 +# ippc19 +# ippc20 +# ippc21 +# ippc22 +# ippc23 +# ippc24 +# ippc25 +# ippc26 +# ippc27 +# ippc28 +# ippc29 +# ippdb00 +# ippdb01 +# ippdb02 +end + +macro check.disks + for i 0 $hosts:n + echo $hosts:$i + exec ll /data/$hosts:$i\.0 + end +end + +if ($SCRIPT) + check.disks + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.dmesg =================================================================== --- trunk/tools/eam/cluster.checks/check.dmesg (revision 41456) +++ trunk/tools/eam/cluster.checks/check.dmesg (revision 41456) @@ -0,0 +1,29 @@ +#!/usr/bin/env mana + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro check.disks + break -auto off + for i 0 $hosts:n + echo $hosts:$i + exec ssh $hosts:$i 'dmesg | grep eth0' + # exec ssh $hosts:$i 'dmesg | grep WARNING | grep -v grep | cut -b -285' + # exec ssh $hosts:$i killall streaksremove + end +end + +macro remote.call + echo $name + exec ssh -n $name 'dmesg | grep responding | prcol 4 | sort | uniq -c' +end + +macro go + break -auto off + node.loop + break -auto on +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.glockfail =================================================================== --- trunk/tools/eam/cluster.checks/check.glockfail (revision 41456) +++ trunk/tools/eam/cluster.checks/check.glockfail (revision 41456) @@ -0,0 +1,44 @@ +#!/usr/bin/env mana + +macro remote.call + if ($0 != 3) + echo "USAGE: remote.call (host1) (host2)" + break + end + + echo "---- $1 ----" + # exec mkdir -p /data/$2\.0/eugene + # exec ssh $1 touch /data/$2\.0/eugene/foobar + break -auto off + exec ssh $1 glockfile -timeout 1 /data/$2\.0/eugene/foobar soft 0.5 + if ($STATUS == 0) + echo "$1 to $2 : $STATUS" + end + break -auto on +end + +macro remote.loop + if ($0 != 2) + echo "USAGE: remote.loop (host1)" + break + end + + local i name + + for i 4 72 + if ($i == 22) continue + sprintf name ipp%03d $i + remote.call $name $1 + end +end + +if ($SCRIPT) + if ($argv:n != 1) + echo "USAGE: check.glockfail (target)" + echo "checks if nodes can set lock on target" + exit 1 + end + + remote.loop $argv:0 + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.glockfile =================================================================== --- trunk/tools/eam/cluster.checks/check.glockfile (revision 41456) +++ trunk/tools/eam/cluster.checks/check.glockfile (revision 41456) @@ -0,0 +1,54 @@ +#!/usr/bin/env mana + +macro remote.call + echo $name + exec ssh $name 'ps aux | grep "glockfile" | grep -v grep | cut -b -285' +end + +macro go + break -auto off + echo "--- storage nodes ---" + for i 4 72 + if ($i == 22) continue + sprintf name ipp%03d $i + remote.call + end + + echo "--- compute nodes ---" + for i 0 64 + if ($i == 0) continue + if ($i == 12) continue + sprintf name ippc%02d $i + remote.call + end + + echo "--- db nodes ---" + for i 0 5 + sprintf name ippdb%02d $i + remote.call + end + + echo "--- atrc nodes ---" + for i 0 4 + sprintf name ippb%02d $i + remote.call + end + + echo "--- stsci nodes ---" + for i 0 20 + sprintf name stsci%02d $i + remote.call + end + + echo "--- stare nodes ---" + for i 0 5 + sprintf name stare%02d $i + remote.call + end + +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.homedir =================================================================== --- trunk/tools/eam/cluster.checks/check.homedir (revision 41456) +++ trunk/tools/eam/cluster.checks/check.homedir (revision 41456) @@ -0,0 +1,19 @@ +#!/usr/bin/env mana + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro remote.call + echo $name + exec ssh -n $name pwd +end + +macro go + break -auto off + node.loop + break -auto on +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.ipp014 =================================================================== --- trunk/tools/eam/cluster.checks/check.ipp014 (revision 41456) +++ trunk/tools/eam/cluster.checks/check.ipp014 (revision 41456) @@ -0,0 +1,98 @@ +#!/usr/bin/env mana + +list hosts +# ipp004 + ipp005 + ipp006 + ipp007 + ipp008 + ipp009 + ipp010 + ipp011 + ipp012 +# ipp013 + ipp014 + ipp015 + ipp016 + ipp017 +# ipp018 + ipp019 + ipp020 + ipp021 + ipp023 + ipp024 + ipp025 + ipp026 + ipp027 + ipp028 + ipp029 + ipp030 + ipp031 + ipp032 + ipp033 + ipp034 + ipp035 + ipp036 + ipp037 + ipp038 + ipp039 + ipp040 + ipp041 + ipp042 + ipp043 + ipp044 + ipp045 + ipp046 + ipp047 + ipp048 + ipp049 + ipp050 + ipp051 + ipp052 + ipp053 + ippc00 + ippc01 + ippc02 + ippc03 + ippc04 + ippc05 + ippc06 + ippc07 + ippc08 + ippc09 + ippc10 + ippc11 + ippc12 + ippc13 + ippc14 + ippc15 + ippc16 + ippc17 + ippc18 + ippc19 + ippc20 + ippc21 + ippc22 + ippc23 + ippc24 + ippc25 + ippc26 + ippc27 + ippc28 + ippc29 + ippdb00 + ippdb01 + ippdb02 +end + +macro check.disks + for i 0 $hosts:n + echo $hosts:$i + exec ssh $hosts:$i ll /data/ipp013.0/ipp/ + end +end + +if ($SCRIPT) + check.disks + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.ipp031 =================================================================== --- trunk/tools/eam/cluster.checks/check.ipp031 (revision 41456) +++ trunk/tools/eam/cluster.checks/check.ipp031 (revision 41456) @@ -0,0 +1,52 @@ +#!/usr/bin/env mana + +macro remote.call + echo $name + exec ssh $name grep ipp031 /etc/mtab +end + +macro go + + echo "--- storage nodes ---" + for i 4 67 + if ($i == 22) continue + sprintf name ipp%03d $i + remote.call + end + + echo "--- compute nodes ---" + for i 0 64 + if ($i == 0) continue + if ($i == 12) continue + sprintf name ippc%02d $i + remote.call + end + + echo "--- db nodes ---" + for i 0 5 + sprintf name ippdb%02d $i + remote.call + end + + echo "--- atrc nodes ---" + for i 0 6 + sprintf name ippb%02d $i + remote.call + end + + for i 0 20 + sprintf name stsci%02d $i + remote.call + end + + echo "--- stare nodes ---" + for i 0 5 + sprintf name stare%02d $i + remote.call + end +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.ipp032 =================================================================== --- trunk/tools/eam/cluster.checks/check.ipp032 (revision 41456) +++ trunk/tools/eam/cluster.checks/check.ipp032 (revision 41456) @@ -0,0 +1,52 @@ +#!/usr/bin/env mana + +macro remote.call + echo $name + exec ssh $name grep ipp032 /etc/mtab +end + +macro go + + echo "--- storage nodes ---" + for i 4 67 + if ($i == 22) continue + sprintf name ipp%03d $i + remote.call + end + + echo "--- compute nodes ---" + for i 0 64 + if ($i == 0) continue + if ($i == 12) continue + sprintf name ippc%02d $i + remote.call + end + + echo "--- db nodes ---" + for i 0 5 + sprintf name ippdb%02d $i + remote.call + end + + echo "--- atrc nodes ---" + for i 0 6 + sprintf name ippb%02d $i + remote.call + end + + for i 0 20 + sprintf name stsci%02d $i + remote.call + end + + echo "--- stare nodes ---" + for i 0 5 + sprintf name stare%02d $i + remote.call + end +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.kill =================================================================== --- trunk/tools/eam/cluster.checks/check.kill (revision 41456) +++ trunk/tools/eam/cluster.checks/check.kill (revision 41456) @@ -0,0 +1,20 @@ +#!/usr/bin/env mana + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro remote.call + echo $name + break -auto off + # exec ssh $name 'sudo -A killall -u ipp' + exec ssh $name 'sudo -A killall mount' + break -auto on +end + +macro go + node.loop +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.localtime =================================================================== --- trunk/tools/eam/cluster.checks/check.localtime (revision 41456) +++ trunk/tools/eam/cluster.checks/check.localtime (revision 41456) @@ -0,0 +1,34 @@ +#!/usr/bin/env mana + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro remote.call + if ($0 != 2) + echo "USAGE: remote.call (host)" + break + end + $host = $1 + + echo host: $host + break -auto off + exec ssh $host md5sum /etc/localtime + break -auto on +end + +macro go + if ($0 != 1) + echo "USAGE: go" + break + end + + node.loop +end + +if ($SCRIPT) + if ($argv:n != 0) + echo "USAGE: check.umount" + exit 1 + end + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.messages =================================================================== --- trunk/tools/eam/cluster.checks/check.messages (revision 41456) +++ trunk/tools/eam/cluster.checks/check.messages (revision 41456) @@ -0,0 +1,60 @@ +#!/usr/bin/env mana + +macro remote.call + # exec ssh $name killall dvo_client + # exec ssh $name ps ux + # exec ssh $name 'ps uxa | grep ippdvo | grep -v grep' + echo "---- $name ----" + exec ssh $name 'ls -l /var/log/messages' + exec ssh $name 'uname -a' +end + +macro go + echo "--- storage nodes ---" + for i 4 67 + if ($i == 22) continue + sprintf name ipp%03d $i + remote.call + end + + echo "--- compute nodes ---" + for i 0 64 + if ($i == 0) continue + if ($i == 12) continue + sprintf name ippc%02d $i + remote.call + end + + echo "--- db nodes ---" + for i 0 5 + sprintf name ippdb%02d $i + remote.call + end + + echo "--- atrc nodes ---" + for i 0 4 + sprintf name ippb%02d $i + remote.call + end + + echo "--- stsci nodes ---" + for i 0 20 + sprintf name stsci%02d $i + remote.call + end + + echo "--- stare nodes ---" + for i 0 5 + sprintf name stare%02d $i + remote.call + end + +end + +# exec ssh $hosts:$i ls -l /var/log/messages +# exec ssh $hosts:$i killall streaksremove + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.mountall.ippx =================================================================== --- trunk/tools/eam/cluster.checks/check.mountall.ippx (revision 41456) +++ trunk/tools/eam/cluster.checks/check.mountall.ippx (revision 41456) @@ -0,0 +1,39 @@ +#!/usr/bin/env mana + +macro remote.call + if ($0 != 2) + echo "USAGE: remote.call (host)" + break + end + $host = $1 + + echo "--- $host ---" + exec ssh $host sudo -A mount -a +end + +macro go + + + echo "--- compute host nodes ---" + for i 1 64 + sprintf name ippc%02d $i + remote.call $name + end + break + + echo "--- xhost nodes ---" + for i 1 104 + if ($i == 71) continue + sprintf name ippx%03d $i + remote.call $name + end +end + +if ($SCRIPT) + if ($argv:n != 0) + echo "USAGE: check.mountall" + exit 1 + end + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.mounts =================================================================== --- trunk/tools/eam/cluster.checks/check.mounts (revision 41456) +++ trunk/tools/eam/cluster.checks/check.mounts (revision 41456) @@ -0,0 +1,28 @@ +#!/usr/bin/env mana + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro remote.call + if ($0 != 2) + echo "USAGE: remote.call (host)" + break + end + $name = $1 + echo "--- $name ---" +# exec ssh -n $name 'grep ippc18.0 /etc/mtab' + exec ssh -n $name 'sudo -A mount -a' +end + +macro go + if ($0 != 1) + echo "USAGE: go" + break + end + + node.loop +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.mounts.ex.cab2 =================================================================== --- trunk/tools/eam/cluster.checks/check.mounts.ex.cab2 (revision 41456) +++ trunk/tools/eam/cluster.checks/check.mounts.ex.cab2 (revision 41456) @@ -0,0 +1,57 @@ +#!/usr/bin/env mana + +macro remote.call + echo "--- $name ---" + exec ssh $name df + # exec ssh $name grep ipp023 /etc/mtab +end + +macro go + echo "--- storage nodes ---" + for i 4 67 + if ($i == 22) continue + if ($i == 28) continue + + sprintf name ipp%03d $i + remote.call + end + + echo "--- compute nodes ---" + for i 0 64 + if ($i == 0) continue + if ($i == 2) continue + if ($i == 12) continue + sprintf name ippc%02d $i + remote.call + end + + echo "--- db nodes ---" + for i 0 5 + sprintf name ippdb%02d $i + remote.call + end + + echo "--- atrc nodes ---" + for i 0 4 + sprintf name ippb%02d $i + remote.call + end + + echo "--- stsci nodes ---" + for i 0 20 + sprintf name stsci%02d $i + remote.call + end + + echo "--- stare nodes ---" + for i 0 5 + sprintf name stare%02d $i + remote.call + end + +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.nfs =================================================================== --- trunk/tools/eam/cluster.checks/check.nfs (revision 41456) +++ trunk/tools/eam/cluster.checks/check.nfs (revision 41456) @@ -0,0 +1,34 @@ +#!/usr/bin/env mana + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro remote.call + if ($0 != 2) + echo "USAGE: remote.call (host)" + break + end + $host = $1 + + echo host: $host + break -auto off + exec ssh $host ll -d /var/lib/nfs/sm + break -auto on +end + +macro go + if ($0 != 1) + echo "USAGE: go" + break + end + + node.loop +end + +if ($SCRIPT) + if ($argv:n != 0) + echo "USAGE: check.umount" + exit 1 + end + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.ping =================================================================== --- trunk/tools/eam/cluster.checks/check.ping (revision 41456) +++ trunk/tools/eam/cluster.checks/check.ping (revision 41456) @@ -0,0 +1,78 @@ +#!/usr/bin/env mana + +macro remote.call + echo $name + # exec ssh $name killall dvo_client + # exec ssh $name ps ux + # exec ssh $name 'ps uxa | grep "^ipp" | grep -v grep' + # exec ssh $name 'ps uxa | grep schast | grep -v grep' + break -auto off + exec ping -c 1 $name + if (not($STATUS)) + echo "$name IS DOWN" + end +end + +macro go + + if (1) + echo "--- storage nodes ---" + for i 4 95 + if ($i == 22) continue + if ($i == 51) continue + sprintf name ipp%03d $i + remote.call + end + + echo "--- compute nodes ---" + for i 0 64 + if ($i == 0) continue + sprintf name ippc%02d $i + remote.call + end + + echo "--- stsci nodes ---" + for i 0 20 + if ($i == 9) continue + sprintf name stsci%02d $i + remote.call + end + + echo "--- stare nodes ---" + for i 0 5 + sprintf name stare%02d $i + remote.call + end + + echo "--- himem nodes ---" + for i 0 15 + sprintf name ipps%02d $i + remote.call + end + end + + echo "--- xhost nodes ---" + for i 1 104 + if ($i == 71) continue + sprintf name ippx%03d $i + remote.call + end + + echo "--- db nodes ---" + for i 0 5 + if ($i == 4) continue + sprintf name ippdb%02d $i + remote.call + end + + echo "--- atrc nodes ---" + for i 0 6 + sprintf name ippb%02d $i + remote.call + end +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.process =================================================================== --- trunk/tools/eam/cluster.checks/check.process (revision 41456) +++ trunk/tools/eam/cluster.checks/check.process (revision 41456) @@ -0,0 +1,101 @@ +#!/usr/bin/env mana + +list hosts + ipp004 + ipp005 + ipp006 + ipp007 + ipp008 + ipp009 + ipp010 + ipp011 + ipp012 + ipp013 + ipp014 + ipp015 + ipp016 + ipp017 + ipp018 + ipp019 + ipp020 + ipp021 + ipp023 + ipp024 + ipp025 + ipp026 + ipp027 + ipp028 + ipp029 + ipp030 + ipp031 + ipp032 + ipp033 + ipp034 + ipp035 + ipp036 + ipp037 + ipp038 + ipp039 + ipp040 + ipp041 + ipp042 + ipp043 + ipp044 + ipp045 + ipp046 + ipp047 + ipp048 + ipp049 + ipp050 + ipp051 + ipp052 + ipp053 + ippc00 + ippc01 + ippc02 + ippc03 + ippc04 + ippc05 + ippc06 + ippc07 + ippc08 + ippc09 + ippc10 + ippc11 + ippc12 + ippc13 + ippc14 + ippc15 + ippc16 + ippc17 + ippc18 + ippc19 + ippc20 + ippc21 + ippc22 + ippc23 + ippc24 + ippc25 + ippc26 + ippc27 + ippc28 + ippc29 + ippdb00 + ippdb01 + ippdb02 +end + +macro check.disks + break -auto off + for i 0 $hosts:n + echo $hosts:$i + exec ssh $hosts:$i ps ux + # exec ssh $hosts:$i 'ps aux | grep "pdflush" | grep -v grep | cut -b -285' + # exec ssh $hosts:$i killall streaksremove + end +end + +if ($SCRIPT) + check.disks + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.psipp =================================================================== --- trunk/tools/eam/cluster.checks/check.psipp (revision 41456) +++ trunk/tools/eam/cluster.checks/check.psipp (revision 41456) @@ -0,0 +1,100 @@ +#!/usr/bin/env mana + +list hosts + ipp004 + ipp005 + ipp006 + ipp007 + ipp008 + ipp009 + ipp010 + ipp011 + ipp012 + ipp013 + ipp014 + ipp015 + ipp016 + ipp017 + ipp018 + ipp019 + ipp020 + ipp021 + ipp023 + ipp024 + ipp025 + ipp026 + ipp027 + ipp028 + ipp029 + ipp030 + ipp031 + ipp032 + ipp033 + ipp034 + ipp035 + ipp036 + ipp037 + ipp038 + ipp039 + ipp040 + ipp041 + ipp042 + ipp043 + ipp044 + ipp045 + ipp046 + ipp047 + ipp048 + ipp049 + ipp050 + ipp051 + ipp052 + ipp053 + ippc00 + ippc01 + ippc02 + ippc03 + ippc04 + ippc05 + ippc06 + ippc07 + ippc08 + ippc09 + ippc10 + ippc11 + ippc12 + ippc13 + ippc14 + ippc15 + ippc16 + ippc17 + ippc18 + ippc19 + ippc20 + ippc21 + ippc22 + ippc23 + ippc24 + ippc25 + ippc26 + ippc27 + ippc28 + ippc29 + ippdb00 + ippdb01 + ippdb02 +end + +macro check.disks + break -auto off + for i 0 $hosts:n + echo $hosts:$i + exec ssh $hosts:$i 'ps aux | grep "^ipp" | grep -v "^ippdvo" | cut -b -285' + # exec ssh $hosts:$i killall streaksremove + end +end + +if ($SCRIPT) + check.disks + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.psux =================================================================== --- trunk/tools/eam/cluster.checks/check.psux (revision 41456) +++ trunk/tools/eam/cluster.checks/check.psux (revision 41456) @@ -0,0 +1,18 @@ +#!/usr/bin/env mana + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro remote.call + echo $name + exec ssh -n $name 'ps uxa | grep -v ^root | grep -v ^apache | grep -v ^nagios | grep -v ^nobody | grep -v ^mysql | grep -v ^bin | grep -v ^1644 | grep -v grep | grep -v ssh' + # exec ssh $name 'ps ux | grep -v ^root | grep -v ^apache | grep -v ^nagios | grep -v ^nobody | grep -v ^mysql | grep -v ^bin | grep -v ^1644 | grep -v grep | grep -v ssh' +end + +macro go + node.loop +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.reboot =================================================================== --- trunk/tools/eam/cluster.checks/check.reboot (revision 41456) +++ trunk/tools/eam/cluster.checks/check.reboot (revision 41456) @@ -0,0 +1,45 @@ +#!/usr/bin/env mana + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro remote.call + if ($0 != 2) + echo "USAGE: remote.call (host)" + break + end + $host = $1 + + echo host: $host + break -auto off + exec echo host: $host >> /home/kiawe/eugene/log.reboots.20160412 + exec ssh -f $host 'last reboot | wc -l >> log.reboots.20160412' + break -auto on +end + +macro go + if ($0 != 1) + echo "USAGE: go" + break + end + + exec rm -f /home/kiawe/eugene/log.reboots.20160412 + node.loop +end + +if ($SCRIPT) + if ($argv:n != 0) + echo "USAGE: check.umount" + exit 1 + end + go + exit 0 +end + +macro node.loop + echo "--- storage nodes ---" + for i 4 98 + sprintf name ipp%03d $i + if ($?skipnodes:$name) continue + remote.call $name + end +end Index: trunk/tools/eam/cluster.checks/check.resolv =================================================================== --- trunk/tools/eam/cluster.checks/check.resolv (revision 41456) +++ trunk/tools/eam/cluster.checks/check.resolv (revision 41456) @@ -0,0 +1,18 @@ +#!/usr/bin/env mana + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro remote.call + echo $name + exec ssh $name sudo -A mv /etc/resolv.conf /etc/resolv.conf.20150506 + exec ssh $name sudo -A cp /home/kiawe/eugene/resolv.conf.ippc19 /etc/resolv.conf +end + +macro go + node.loop +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.sar =================================================================== --- trunk/tools/eam/cluster.checks/check.sar (revision 41456) +++ trunk/tools/eam/cluster.checks/check.sar (revision 41456) @@ -0,0 +1,111 @@ +#!/usr/bin/env mana + +list hosts + ipp004 + ipp005 + ipp006 + ipp007 + ipp008 + ipp009 + ipp010 + ipp011 + ipp012 + ipp013 + ipp014 +# ipp015 + ipp016 + ipp017 + ipp018 + ipp019 + ipp020 + ipp021 + ipp023 + ipp024 + ipp025 + ipp026 + ipp027 + ipp028 + ipp029 + ipp030 + ipp031 + ipp032 + ipp033 + ipp034 + ipp035 + ipp036 + ipp037 + ipp038 + ipp039 + ipp040 + ipp041 + ipp042 + ipp043 + ipp044 + ipp045 + ipp046 + ipp047 + ipp048 + ipp049 + ipp050 + ipp051 + ipp052 + ipp053 + ipp054 + ipp055 + ipp056 + ipp057 + ipp058 + ipp059 + ipp060 + ipp061 + ipp062 + ipp063 + ipp064 + ipp065 + ipp066 +# ippc00 +# ippc01 +# ippc02 +# ippc03 +# ippc04 +# ippc05 +# ippc06 +# ippc07 +# ippc08 +# ippc09 +# ippc10 +# ippc11 +# ippc12 +# ippc13 +# ippc14 +# ippc15 +# ippc16 +# ippc17 +# ippc18 +# ippc19 +# ippc20 +# ippc21 +# ippc22 +# ippc23 +# ippc24 +# ippc25 +# ippc26 +# ippc27 +# ippc28 +# ippc29 +# ippdb00 +# ippdb01 +# ippdb02 +end + +macro check.disks + for i 0 $hosts:n + echo $hosts:$i + exec ssh -n -f $hosts:$i 'sar 1 10 > $hosts:$i.sar.dat' + end +end + +if ($SCRIPT) + check.disks + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.scidbm =================================================================== --- trunk/tools/eam/cluster.checks/check.scidbm (revision 41456) +++ trunk/tools/eam/cluster.checks/check.scidbm (revision 41456) @@ -0,0 +1,84 @@ +#!/usr/bin/env mana + +macro remote.call + echo $name + break -auto off +# exec ssh -n $name ping -c 1 ippb20 + exec ssh -n $name ping -c 1 scidbm + exec ssh -n $name ls -l /run/systemd/resolve/stub-resolv.conf +# exec ssh -n $name 'ps aux | grep resolv | grep -v grep' +# exec ssh -n $name sudo -A service systemd-resolved restart + if (not($STATUS)) + echo "$name IS DOWN" + end +end + +macro go + + for i 128 243 + sprintf name ippc%03d $i + remote.call + end + break + + if (1) + echo "--- storage nodes ---" + for i 4 95 + if ($i == 22) continue + if ($i == 51) continue + sprintf name ipp%03d $i + remote.call + end + + echo "--- compute nodes ---" + for i 0 64 + if ($i == 0) continue + sprintf name ippc%02d $i + remote.call + end + + echo "--- stsci nodes ---" + for i 0 20 + if ($i == 9) continue + sprintf name stsci%02d $i + remote.call + end + + echo "--- stare nodes ---" + for i 0 5 + sprintf name stare%02d $i + remote.call + end + + echo "--- himem nodes ---" + for i 0 15 + sprintf name ipps%02d $i + remote.call + end + end + + echo "--- xhost nodes ---" + for i 1 104 + if ($i == 71) continue + sprintf name ippx%03d $i + remote.call + end + + echo "--- db nodes ---" + for i 0 5 + if ($i == 4) continue + sprintf name ippdb%02d $i + remote.call + end + + echo "--- atrc nodes ---" + for i 0 6 + sprintf name ippb%02d $i + remote.call + end +end + +if ($SCRIPT) + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.speed =================================================================== --- trunk/tools/eam/cluster.checks/check.speed (revision 41456) +++ trunk/tools/eam/cluster.checks/check.speed (revision 41456) @@ -0,0 +1,53 @@ +#!/usr/bin/env mana + +$skipnodes:ippx070 = 1 + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro node.loop + if ($0 != 1) + echo "USAGE: node.loop" + echo "expects to call remote.call" + break + end + + echo "--- x nodes ---" + for i 1 104 + sprintf name ippx%03d $i + if ($?skipnodes:$name) continue + remote.call $name + end + break +end + +macro remote.call + if ($0 != 2) + echo "USAGE: remote.call (host)" + break + end + $host = $1 + + # echo host: $host + break -auto off + $Nsec = `ssh $host speedcheck.sh` + echo $host $Nsec + break -auto on +end + +macro go + if ($0 != 1) + echo "USAGE: go" + break + end + + node.loop +end + +if ($SCRIPT) + if ($argv:n != 0) + echo "USAGE: check.umount" + exit 1 + end + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.speed.highc =================================================================== --- trunk/tools/eam/cluster.checks/check.speed.highc (revision 41456) +++ trunk/tools/eam/cluster.checks/check.speed.highc (revision 41456) @@ -0,0 +1,81 @@ +#!/usr/bin/env mana + +$skipnodes:ippc102 = 1 +$skipnodes:ippc122 = 1 +$skipnodes:ippc123 = 1 +$skipnodes:ippc127 = 1 + +$srcdir:0 = /data/ipp105.0/eugene/taurus.20160917/catdir.001 +$srcdir:1 = /data/ipp106.0/eugene/taurus.20160917/catdir.002 +$srcdir:2 = /data/ipp107.0/eugene/taurus.20160917/catdir.003/ +$srcdir:3 = /data/ipp108.0/eugene/taurus.20160917/catdir.004/ +$srcdir:4 = /data/ipp109.0/eugene/taurus.20160917/catdir.005/ +$srcdir:5 = /data/ipp110.0/eugene/taurus.20160917/catdir.006/ +$srcdir:6 = /data/ipp111.0/eugene/taurus.20160917/catdir.007/ +$srcdir:7 = /data/ipp112.0/eugene/taurus.20160917/catdir.008/ +$srcdir:8 = /data/ipp113.0/eugene/taurus.20160917/catdir.009/ +$srcdir:9 = /data/ipp114.0/eugene/taurus.20160917/catdir.010/ +$srcdir:10 = /data/ipp115.0/eugene/taurus.20160917/catdir.011/ +$srcdir:11 = /data/ipp116.0/eugene/taurus.20160917/catdir.012/ +$srcdir:12 = /data/ipp117.0/eugene/taurus.20160917/catdir.013/ + +$TIMEFORMAT = mjd + +macro node.loop + if ($0 != 1) + echo "USAGE: node.loop" + echo "expects to call remote.call" + break + end + + echo "--- c nodes ---" + + $S0 = 66 + 5*($set + 0) + $S1 = min (66 + 5*($set + 1) , 128) + for i $S0 $S1 + sprintf name ippc%d $i + if ($?skipnodes:$name) continue + remote.call $name + end + break +end + +macro remote.call + if ($0 != 2) + echo "USAGE: remote.call (host)" + break + end + $host = $1 + + # echo host: $host + break -auto off + # speedcheck + # $Nsec = `ssh $host speedcheck.sh` + # echo $host $Nsec + + # local df check + # exec ssh $host df -l + + exec ssh -n $host sudo -A chmod 777 /export/$host.0 + exec ssh -n $host sudo -A chmod 777 /export/$host.1 + + ctimes -abs now -var t0 + exec ssh -n $host "rsync -auv /data/ipp105.0/eugene/taurus.20160917/catdir.001/ /export/$host.0/test.io.20170217.v0/ >& log.$host.rsync.v0" + ctimes -abs now -var t1 + exec ssh -n $host "rsync -auv /data/ipp105.0/eugene/taurus.20160917/catdir.001/ /export/$host.1/test.io.20170217.v0/ >& log.$host.rsync.v1" + ctimes -abs now -var t2 + + echo $host : {86400*($t1-$t0)} : {86400*($t2-$t1)} + + break -auto on +end + +if ($SCRIPT) + if ($argv:n != 1) + echo "USAGE: check.speed.highc (set)" + exit 1 + end + $set = $argv:0 + node.loop + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.stsci =================================================================== --- trunk/tools/eam/cluster.checks/check.stsci (revision 41456) +++ trunk/tools/eam/cluster.checks/check.stsci (revision 41456) @@ -0,0 +1,74 @@ +#!/usr/bin/env mana + +list mynodes + ipp021 + ipp023 + ipp024 + ipp025 + ipp026 + ipp027 + ipp028 + ipp031 + ipp041 + ipp053 + ipp054 + ipp055 + ipp056 + ipp057 + ipp058 + ipp059 + ipp063 + ipp065 + ipp068 + ippc43 +end + +macro my.node.loop + if ($0 != 1) + echo "USAGE: node.loop" + echo "expects to call remote.call" + break + end + + for i 0 $mynodes:n + remote.call $mynodes:$i + end +end + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro remote.call + if ($0 != 2) + echo "USAGE: remote.call (host)" + break + end + $host = $1 + + echo host: $host + break -auto off + exec ssh $host "grep stsci /etc/mtab" + if ($STATUS) + echo "$host has stsci" + end +# exec ssh $host sudo -A killall df +# exec ssh -f $host src/bin/shell/umount.stsci + break -auto on +end + +macro go + if ($0 != 1) + echo "USAGE: go" + break + end + + node.loop +end + +if ($SCRIPT) + if ($argv:n != 0) + echo "USAGE: check.stsci" + exit 1 + end + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.umount =================================================================== --- trunk/tools/eam/cluster.checks/check.umount (revision 41456) +++ trunk/tools/eam/cluster.checks/check.umount (revision 41456) @@ -0,0 +1,34 @@ +#!/usr/bin/env mana + +input /home/kiawe/eugene/src/bin/shell/node.loop + +macro remote.call + if ($0 != 2) + echo "USAGE: remote.call (host)" + break + end + $host = $1 + + echo "--- $name ---" + exec ssh -f $host "sudo -A src/bin/shell/my.alt.force.umount $myHost $dir >& log.umount.$host.$myHost &" +end + +macro go + if ($0 != 3) + echo "USAGE: go (dir)" + break + end + $myHost = $1 + $dir = $2 + + node.loop +end + +if ($SCRIPT) + if ($argv:n != 2) + echo "USAGE: check.umount (host) (dir)" + exit 1 + end + go $argv:0 $argv:1 + exit 0 +end Index: trunk/tools/eam/cluster.checks/check.umountall =================================================================== --- trunk/tools/eam/cluster.checks/check.umountall (revision 41456) +++ trunk/tools/eam/cluster.checks/check.umountall (revision 41456) @@ -0,0 +1,77 @@ +#!/usr/bin/env mana + +macro remote.call + if ($0 != 2) + echo "USAGE: remote.call (host)" + break + end + $host = $1 + + exec ssh -f "$host sudo -A umount -a >& log.umount.$host" & +end + + +macro go + if ($0 != 1) + echo "USAGE: go" + break + end + + echo "--- storage nodes ---" + for i 4 96 + if ($i == 6) continue + if ($i == 22) continue + if ($i == 64) continue + if ($i == 94) continue + sprintf name ipp%03d $i + remote.call $name + end + break + + echo "--- compute nodes ---" + for i 0 64 + if ($i == 0) continue + sprintf name ippc%02d $i + remote.call $name + end + + echo "--- stsci nodes ---" + for i 0 20 + sprintf name stsci%02d $i + remote.call $name + end + + echo "--- x nodes ---" + for i 1 104 + if ($i == 71) continue + sprintf name ippx%03d $i + remote.call $name + end + + echo "--- db nodes ---" + for i 0 7 + sprintf name ippdb%02d $i + remote.call $name + end + + echo "--- atrc nodes ---" + for i 0 4 + sprintf name ippb%02d $i + remote.call $name + end + + # echo "--- stare nodes ---" + for i 0 5 + sprintf name stare%02d $i + remote.call $name + end +end + +if ($SCRIPT) + if ($argv:n != 0) + echo "USAGE: check.umount" + exit 1 + end + go + exit 0 +end Index: trunk/tools/eam/cluster.checks/my.alt.force.umount =================================================================== --- trunk/tools/eam/cluster.checks/my.alt.force.umount (revision 41456) +++ trunk/tools/eam/cluster.checks/my.alt.force.umount (revision 41456) @@ -0,0 +1,10 @@ +#!/bin/csh -f + +if ($#argv != 2) then + echo "USAGE: force.umount (ipphost) (volume)" + exit 2 +endif + +umount -f -v /data/$2 +umount -f -v /data/$2 +umount -f -v /data/$2 Index: trunk/tools/eam/cluster.checks/node.loop =================================================================== --- trunk/tools/eam/cluster.checks/node.loop (revision 41456) +++ trunk/tools/eam/cluster.checks/node.loop (revision 41456) @@ -0,0 +1,107 @@ + +# retired b-nodes: +$skipnodes:ippb00 = 1 +$skipnodes:ippb01 = 1 +$skipnodes:ippb02 = 1 +$skipnodes:ippb03 = 1 +$skipnodes:ippb04 = 1 +$skipnodes:ippb05 = 1 + +# never existed: +$skipnodes:ipp098 = 1 +$skipnodes:ipp099 = 1 + +# retired c-nodes +$skipnodes:ippc20 = 1 +$skipnodes:ippc21 = 1 +$skipnodes:ippc22 = 1 + +# bad / dead c-nodes: +$skipnodes:ippc27 = 1 +$skipnodes:ippc37 = 1 +$skipnodes:ippc55 = 1 +$skipnodes:ippc58 = 1 +$skipnodes:ippc61 = 1 +$skipnodes:ippc66 = 1 +$skipnodes:ippc67 = 1 +$skipnodes:ippc70 = 1 +$skipnodes:ippc72 = 1 +$skipnodes:ippc94 = 1 +$skipnodes:ippc97 = 1 +$skipnodes:ippc112 = 1 +$skipnodes:ippc122 = 1 +$skipnodes:ippc123 = 1 +$skipnodes:ippc102 = 1 +$skipnodes:ippc107 = 1 +$skipnodes:ippc122 = 1 +$skipnodes:ippc123 = 1 +$skipnodes:ippc125 = 1 +$skipnodes:ippc127 = 1 + +# bad / dead x-nodes: +$skipnodes:ippx065 = 1 +$skipnodes:ippx084 = 1 + +# non-existent db nodes: +$skipnodes:ippdb02 = 1 +$skipnodes:ippdb03 = 1 +$skipnodes:ippdb04 = 1 +$skipnodes:ippdb07 = 1 + +macro node.loop + if ($0 != 1) + echo "USAGE: node.loop" + echo "expects to call remote.call" + break + end + + echo "--- storage nodes ---" + for i 32 130 + sprintf name ipp%03d $i + if (($i > 32) && ($i < 54)) continue + if ($?skipnodes:$name) continue + remote.call $name + end + + echo "--- compute nodes ---" + for i 19 242 + sprintf name ippc%02d $i + if ($?skipnodes:$name) continue + remote.call $name + end + + echo "--- x nodes ---" + for i 1 89 + sprintf name ippx%03d $i + if ($?skipnodes:$name) continue + remote.call $name + end + + echo "--- db nodes ---" + for i 1 9 + sprintf name ippdb%02d $i + if ($?skipnodes:$name) continue + remote.call $name + end + + echo "--- s nodes ---" + for i 0 15 + sprintf name ipps%02d $i + if ($?skipnodes:$name) continue + remote.call $name + end + + echo "--- stare nodes ---" + for i 0 5 + sprintf name stare%02d $i + if ($?skipnodes:$name) continue + remote.call $name + end + + echo "--- atrc nodes ---" + for i 4 23 + sprintf name ippb%02d $i + if ($?skipnodes:$name) continue + remote.call $name + end +end Index: trunk/tools/eam/rebalance/rebalance.mana =================================================================== --- trunk/tools/eam/rebalance/rebalance.mana (revision 41455) +++ trunk/tools/eam/rebalance/rebalance.mana (revision 41456) @@ -40,8 +40,8 @@ # generate the full paths: $source = /data/$srchost.$srcvol/nebulous - $target = /data/$tgthost.$tgtvol/rebalance/$srchost.$srcvol/nebulous + # $target = /data/$tgthost.$tgtvol/rebalance/$srchost.$srcvol/nebulous ## make sure target exists - mkdir $target + # mkdir $target ## this script must be run on the same machine as the source @@ -106,5 +106,5 @@ ## make sure target exists - mkdir $target + # mkdir $target ## this script must be run on the same machine as the source @@ -159,4 +159,9 @@ sprintf movelog %s/rebalance/movefiles.%s.log $srcdir $VERSION sprintf subdirset "%s/??" $srcdir + + local isFound + file $movelist isFound + if ($isFound) continue + exec find $subdirset -size +$BIGSIZE > $filelist @@ -216,4 +221,17 @@ $target = /data/$tgthost.$tgtvol/rebalance/$srchost.$srcvol/nebulous + ## make sure target exists + mkdir $target + + $TIMEFORMAT = mjd + $TIMEREF = 2000/01/01,00:00:00; # this is not used but required to exist for ctimes + + # give the start of the night in HST hours: + $HSTNightStart = 18.5 + $HSTNightEnd = 7.5 + + $UTNightStart = ($HSTNightStart + 10) / 24; $UTNightStart = $UTNightStart - int($UTNightStart) + $UTNightEnd = ($HSTNightEnd + 10) / 24; $UTNightEnd = $UTNightEnd - int($UTNightEnd) + for i 0 $NSUBDIR sprintf srcdir "%s/%02x" $source $i @@ -226,5 +244,19 @@ echo "rsyncing $srcdir/ to $tgtdir/ : $movelog" + break -auto off exec rsync -auv --files-from=$movelist $srcdir/ $tgtdir/ >& $movelog + break -auto on + + # check the time: is it night time (sleep and restart later) + $isNight = 1 + while ($isNight) + ctimes -abs now -var tnow + $tfrac = $tnow - int($tnow) + $isNight = (($tfrac > $UTNightStart) && ($tfrac < $UTNightEnd)) + if ($isNight) + echo "it is night time, sleeping until the morning" + sleep 600 + end + end end end Index: trunk/tools/eam/triples/find.triples.pl =================================================================== --- trunk/tools/eam/triples/find.triples.pl (revision 41456) +++ trunk/tools/eam/triples/find.triples.pl (revision 41456) @@ -0,0 +1,80 @@ +#! /usr/bin/env perl + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +## read command-line options +my ($so_min, $so_max); + +GetOptions('so_min=s' => \$so_min, 'so_max=s' => \$so_max) || pod2usage(2); + +## set up Nebulous connection +my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser ); +my $conffile = '/data/ippc19.0/home/eugene/rebalance.20200605/nebulous.config'; + +if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "$dbname $dbhost $dbport $dbuser $dbpass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless + defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass; + +pod2usage( -msg => "Cannot determine so_min or so_max", -exitval => 2) unless + defined $so_min && defined $so_max; + +my $dsn; +if ($dbport eq "NONE") { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost"; +} else { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport"; +} + +my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "ERROR : Unable to connect to database $DBI::errstr\n"; + +my $query = "SELECT ext_id FROM (SELECT so_id, ins_id, ext_id, count(ins_id) AS Nins FROM storage_object JOIN instance USING (so_id) WHERE (so_id >= $so_min) AND (so_id < $so_max) GROUP BY so_id) AS tmp WHERE (Nins = 3)"; + +print "# query: $query\n"; + +my $rowref = $dbh->selectall_arrayref( $query ); +my @array = @$rowref; +my $Nrows = @array; + +if ($Nrows == 0) { # We should find this instance + print "ERROR : NO ENTRIES\n"; + exit 3; +} + +print "# found $Nrows entries\n"; + +for (my $i = 0; $i < $Nrows; $i++) { + + my $valref = $array[$i]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 1) { + print "ERROR : INVALID DB RESPONSE ($i)\n"; + next; + } + + my $ext_id = $values[0]; + + print "$ext_id\n"; +} + Index: trunk/tools/eam/triples/triples.pl =================================================================== --- trunk/tools/eam/triples/triples.pl (revision 41456) +++ trunk/tools/eam/triples/triples.pl (revision 41456) @@ -0,0 +1,192 @@ +#! /usr/bin/env perl + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +## read command-line options +my ($listing); + +GetOptions('listing=s' => \$listing) || pod2usage(2 ); + +## set up Nebulous connection +my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser ); +my $conffile = '/data/ippc19.0/home/eugene/rebalance.20200605/nebulous.config'; + +if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; } + } + close(IN); + # print STDERR "$dbname $dbhost $dbport $dbuser $dbpass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless + defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass; + +pod2usage( -msg => "specify list of --listing=FILENAME", -exitval => 2) unless defined $listing; + +my $dsn; +if ($dbport eq "NONE") { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost"; +} else { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport"; +} + +my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "ERROR : Unable to connect to database $DBI::errstr\n"; + +## get the list of volumes by vol_id + +my @volnames; +my @volxattr; +&get_volnames(); + +## get a list of the nebfiles to cull (how is this generated? what is the format?) + +# assume a file with the format: nebfile ?? +# open (FILE, "triples.dat"); +open (FILE, $listing); +my @mylist = ; +close (FILE); + +# select * from (select so_id, ins_id, ext_id, count(ins_id) as Nins from storage_object join instance using (so_id) where (so_id >= 4200000000) and (so_id < 4200100000) and (ext_id like "%.fits") group by so_id) as tmp where (Nins = 3) limit 50; + + +foreach my $nebname (@mylist) { + chomp ($nebname); + + if ($nebname =~ m|^#|) { next; } + + + # check that this instance exists in DB (if not, need to add it) + my $query = "SELECT uri, ins_id, vol_id FROM instance join storage_object using (so_id) WHERE ext_id = '$nebname'"; + + my $rowref = $dbh->selectall_arrayref( $query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We should find this instance + print "ERROR : NO DB ENTRY: $nebname\n"; + next; + } + + if ($Nrows != 3) { # We should always find 3 instances (these are selected to be triples) + print "ERROR : NOT A TRIPLE: $nebname\n"; + next; + } + + ## print "NROWS: $Nrows\n"; + ## foreach my $valref (@array) { + ## my @values = @$valref; + ## foreach my $value (@values) { + ## print "VAL: $value, "; + ## } + ## print "\n"; + ## } + + my @sizes; + my $hasBackup = 0; + my $hasPrimary = 0; + my $Nvalid = 0; + my $sizesMatch = 0; + + for (my $i = 0; $i < $Nrows; $i++) { + + my $valref = $array[$i]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 3) { + print "ERROR : INVALID DB RESPONSE: $nebname\n"; + goto skip; + } + + my $uri = $values[0]; + my $vol_id = $values[2]; + +# ^[0-9a-fA-F][0-9a-fA-F]/(\d+)\..*|; + + my ($uri_actual) = $uri =~ m|^file://(.*)|; + # print "uri_actual: $uri_actual\n"; + if ($uri_actual eq "") { + print "ERROR: invalid uri format $uri\n"; + goto skip; + } + + # rather than split the filename, perhaps I should use the vol_id + ## my ($uri_host) = $uri_actual =~ m|/data/(ipp\d\d\d).*|; + ## if ($uri_actual eq "") { + ## print "ERROR: invalid uri format $uri_actual\n"; + ## goto skip; + ## } + + my $uri_host = $volnames[$vol_id]; + if ($volxattr[$vol_id] == 0) { $hasPrimary = 1; } + if ($volxattr[$vol_id] == 1) { $hasBackup = 1; } + + if (($volxattr[$vol_id] == 0) || ($volxattr[$vol_id] == 1)) { $Nvalid ++; } + + my @statarray = stat($uri_actual); + + $sizes[$i] = $statarray[7]; + # print "found $uri_actual on volume $vol_id = $uri_host, size is $statarray[7]\n"; + } + + if (($sizes[0] == $sizes[1]) && ($sizes[0] == $sizes[2])) { $sizesMatch = 1; } + + if ($sizesMatch && $hasPrimary && !$hasBackup && ($Nvalid == 3)) { + print "REDX: $nebname $sizes[0] $sizesMatch $hasPrimary $hasBackup $Nvalid\n"; + next; + } + if ($sizesMatch && $hasPrimary && $hasBackup && ($Nvalid == 3)) { + print "CULL: $nebname $sizes[0] $sizesMatch $hasPrimary $hasBackup $Nvalid\n"; + next; + } + + print "SKIP: $nebname $sizes[0] $sizesMatch $hasPrimary $hasBackup $Nvalid\n"; + skip: +} + +sub get_volnames () { + + # check that this instance exists in DB (if not, need to add it) + my $vol_query = "SELECT vol_id, host, xattr FROM volume"; + + my $rowref = $dbh->selectall_arrayref( $vol_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We should find this instance + print "ERROR : missing volumes?\n"; + next; + } + + for (my $i = 0; $i < $Nrows; $i++) { + + my $valref = $array[$i]; + my @values = @$valref; + + my $Nvalue = @values; + + my $vol_id = $values[0]; + my $volname = $values[1]; + my $xattr = $values[2]; + + # print "volume: $volname = $vol_id\n"; + + $volnames[$vol_id] = $volname; + $volxattr[$vol_id] = $xattr; + } +} +