Skip to content

Commit

Permalink
kubectl-ko: add support for tracing nodes (#2697)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed Apr 25, 2023
1 parent f5fee4c commit 53bfcf4
Show file tree
Hide file tree
Showing 3 changed files with 192 additions and 99 deletions.
161 changes: 91 additions & 70 deletions dist/images/kubectl-ko
View file
Expand Up @@ -22,6 +22,8 @@ showHelp(){
echo " trace ... trace ovn microflow of specific packet"
echo " trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port] trace ICMP/TCP/UDP"
echo " trace {namespace/podname} {target ip address} [target mac address] arp {request|reply} trace ARP request/reply"
echo " trace {node//nodename} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port] trace ICMP/TCP/UDP"
echo " trace {node//nodename} {target ip address} [target mac address] arp {request|reply} trace ARP request/reply"
echo " diagnose {all|node} [nodename] diagnose connectivity of all nodes or a specific node"
echo " env-check check the environment configuration"
echo " tuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version] deploy kernel optimisation components to the system"
Expand Down Expand Up @@ -172,98 +174,110 @@ tcpdump(){

trace(){
set +u
namespacedPod="$1"
namespace=$(echo "$namespacedPod" | cut -d "/" -f1)
podName=$(echo "$namespacedPod" | cut -d "/" -f2)
if [ "$podName" = "$namespacedPod" ]; then
namespace="default"
local lsp= namespace= node= typedName= optNamespace=
if [[ "$1" =~ .*//.* ]]; then
node=${1##*/}
typedName="node/$node"
lsp=$(kubectl get $typedName -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/port_name})
else
namespacedPod="$1"
namespace=$(echo "$namespacedPod" | cut -d "/" -f1)
podName=$(echo "$namespacedPod" | cut -d "/" -f2)
if [ "$podName" = "$namespacedPod" ]; then
namespace="default"
fi
typedName="pod/$podName"
optNamespace="-n $namespace"
lsp="$podName.$namespace"
node=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})
fi

dst="$2"
local dst="$2"
if [ -z "$dst" ]; then
echo "Error: missing target IP address"
exit 1
fi

hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
if [ "$hostNetwork" = "true" ]; then
echo "Can not trace host network pod"
exit 1
fi

af="4"
nw="nw"
proto=""
if [[ "$dst" =~ .*:.* ]]; then
af="6"
nw="ipv6"
proto="6"
fi

dstMac=""
local dstMac=
if echo "$3" | grep -qE '^([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}$'; then
dstMac=$3
shift
fi

type="$3"
local type="$3"
if [ -z "$type" ]; then
echo "Error: missing protocol"
echo "Usage:"
echo " kubectl ko trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]"
echo " kubectl ko trace {namespace/podname} {target ip address} [target mac address] arp {request|reply}"
echo " kubectl ko trace {node//nodename} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]"
echo " kubectl ko trace {node//nodename} {target ip address} [target mac address] arp {request|reply}"
exit 1
fi

if [ ! -z $namespace ]; then
hostNetwork=$(kubectl get "$typedName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
if [ "$hostNetwork" = "true" ]; then
echo "Can not trace host network pod"
exit 1
fi
fi

local af="4" nw="nw" proto=
if [[ "$dst" =~ .*:.* ]]; then
af="6"
nw="ipv6"
proto="6"
fi

if [ "$type" = "arp" ]; then
if [ $af -eq 6 ]; then
echo "Error: invalid target IP address: $dst"
exit 1
fi
fi

podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath="{.status.podIPs[*].ip}"))
if [ ${#podIPs[@]} -eq 0 ]; then
podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address} | sed 's/,/ /g'))
if [ ${#podIPs[@]} -eq 0 ]; then
echo "Error: Pod address not ready"
exit 1
fi
local ips=($(kubectl get "$typedName" $optNamespace -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address} | sed 's/,/ /g'))
if [ ${#ips[@]} -eq 0 ]; then
echo "Error: $typedName address not ready"
exit 1
fi
if [ ! -n "$namespace" -a "$type" != "arp" ]; then
ips=($(kubectl get $typedName -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') ${ips[@]})
fi

podIP=""
for ip in ${podIPs[@]}; do
local srcIP=""
for ip in ${ips[@]}; do
if [ "$af" = "4" ]; then
if [[ ! "$ip" =~ .*:.* ]]; then
podIP=$ip
srcIP=$ip
break
fi
elif [[ "$ip" =~ .*:.* ]]; then
podIP=$ip
srcIP=$ip
break
fi
done

if [ -z "$podIP" ]; then
echo "Error: Pod $namespacedPod has no IPv$af address"
if [ -z "$srcIP" ]; then
echo "Error: $typedName has no IPv$af address"
exit 1
fi

nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})
ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}')
local ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$node'")].metadata.name}')
if [ -z "$ovnCni" ]; then
echo "Error: no kube-ovn-cni Pod running on node $nodeName"
exit 1
fi

ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
local ls=$(kubectl get "$typedName" $optNamespace -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
if [ -z "$ls" ]; then
echo "Error: Pod address not ready"
echo "Error: $typedName address not ready"
exit 1
fi

local cidr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/cidr})
mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
local cidr=$(kubectl get "$typedName" $optNamespace -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/cidr})
local mac=$(kubectl get "$typedName" $optNamespace -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})

if [ "$type" != "arp" -o "$4" = "reply" ]; then
# need destination mac
Expand All @@ -286,8 +300,8 @@ trace(){
exit 1
fi

vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})
logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway})
local vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})
local logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway})
if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then
gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway})
if [[ "$gateway" =~ .*,.* ]]; then
Expand All @@ -298,40 +312,45 @@ trace(){
fi
fi

nicName=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r')
local nicName=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$lsp" | tr -d '\r')
if [ -z "$nicName" ]; then
echo "Error: failed to find ovs interface for Pod namespacedPod on node $nodeName"
echo "Error: failed to find ovs interface for LSP $lsp"
exit 1
fi

podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type})
podNetNs=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//')
local podNicType=$(kubectl get "$typedName" $optNamespace -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type})
local podNetNs=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//')
local nicName= nsenterCmd=
if [ ! -z $podNetNs ]; then
nsenterCmd="nsenter --net='$podNetNs'"
fi
if [ "$podNicType" != "internal-port" ]; then
interface=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=name find interface external_id:iface-id="$podName"."$namespace")
peer=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ip link show $interface | grep -oE "^[0-9]+:\\s$interface@if[0-9]+" | awk -F @ '{print $2}')
peerIndex=${peer//if/}
peer=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ip link show type veth | grep "^$peerIndex:" | awk -F @ '{print $1}')
local interface=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=name find interface external_id:iface-id="$lsp")
local peer=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ip link show $interface | grep -oE "^[0-9]+:\\s$interface@if[0-9]+" | awk -F @ '{print $2}')
local peerIndex=${peer//if/}
local peer=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- $nsenterCmd ip link show type veth | grep "^$peerIndex:" | awk -F @ '{print $1}')
nicName=$(echo $peer | awk '{print $2}')
fi

set +o pipefail
master=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ip link show $nicName | grep -Eo '\smaster\s\w+\s' | awk '{print $2}')
local master=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- $nsenterCmd ip link show $nicName | grep -Eo '\smaster\s\w+\s' | awk '{print $2}')
set -o pipefail
if [ ! -z "$master" ]; then
echo "Error: Pod nic $nicName is a slave of $master, please set the destination mac address."
exit 1
fi

local cmd= output=
if [[ "$gateway" =~ .*:.* ]]; then
cmd="ndisc6 -q $gateway $nicName"
output=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName")
output=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- $nsenterCmd ndisc6 -q "$gateway" "$nicName")
else
cmd="arping -c3 -C1 -i1 -I $nicName $gateway"
output=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway")
output=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- $nsenterCmd arping -c3 -C1 -i1 -I "$nicName" "$gateway")
fi

if [ $? -ne 0 ]; then
echo "Error: failed to execute '$cmd' in Pod's netns"
echo "Error: failed to execute '$cmd' in $typedName"
exit 1
fi

Expand All @@ -341,7 +360,7 @@ trace(){

if [ -z "$dstMac" ]; then
echo "Using the gateway mac address as destination"
lr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router})
local lr=$(kubectl get "$typedName" $optNamespace -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router})
if [ -z "$lr" ]; then
lr=$(kubectl get subnet "$ls" -o jsonpath={.spec.vpc})
fi
Expand All @@ -359,44 +378,44 @@ trace(){
dstMac="ff:ff:ff:ff:ff:ff"
fi

lsp="$podName.$namespace"
lspUUID=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=_uuid find logical_switch_port name="$lsp")
local lspUUID=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=_uuid find logical_switch_port name="$lsp")
if [ -z "$lspUUID" ]; then
echo "Notice: LSP $lsp does not exist"
fi
vmOwner=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath='{.metadata.ownerReferences[?(@.kind=="VirtualMachineInstance")].name}')
local vmOwner=$(kubectl get "$typedName" $optNamespace -o jsonpath='{.metadata.ownerReferences[?(@.kind=="VirtualMachineInstance")].name}')
if [ ! -z "$vmOwner" ]; then
lsp="$vmOwner.$namespace"
fi

if [ -z "$lsp" ]; then
echo "Error: failed to get LSP of Pod $namespace/$podName"
echo "Error: failed to get LSP of $typedName"
exit 1
fi

case $type in
icmp)
set -x
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace "$ls" "inport == \"$lsp\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst && ct.new"
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace "$ls" "inport == \"$lsp\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $srcIP && eth.dst == $dstMac && ip$af.dst == $dst && ct.new"
;;
tcp|udp)
set -x
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace "$ls" "inport == \"$lsp\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4 && ct.new"
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace "$ls" "inport == \"$lsp\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $srcIP && eth.dst == $dstMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4 && ct.new"
;;
arp)
case "$4" in
""|request)
set -x
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace "$ls" "inport == \"$lsp\" && eth.src == $mac && eth.dst == $dstMac && arp.op == 1 && arp.sha == $mac && arp.tha == 00:00:00:00:00:00 && arp.spa == $podIP && arp.tpa == $dst"
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace "$ls" "inport == \"$lsp\" && eth.src == $mac && eth.dst == $dstMac && arp.op == 1 && arp.sha == $mac && arp.tha == 00:00:00:00:00:00 && arp.spa == $srcIP && arp.tpa == $dst"
;;
reply)
set -x
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace "$ls" "inport == \"$lsp\" && eth.src == $mac && eth.dst == $dstMac && arp.op == 2 && arp.sha == $mac && arp.tha == $dstMac && arp.spa == $podIP && arp.tpa == $dst"
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace "$ls" "inport == \"$lsp\" && eth.src == $mac && eth.dst == $dstMac && arp.op == 2 && arp.sha == $mac && arp.tha == $dstMac && arp.spa == $srcIP && arp.tpa == $dst"
;;
*)
echo "Error: invalid ARP type $4"
echo "Usage:"
echo " kubectl ko trace {namespace/podname} {target ip address} [target mac address] arp {request|reply}"
echo " kubectl ko trace {node//nodename} {target ip address} [target mac address] arp {request|reply}"
exit 1
;;
esac
Expand All @@ -406,6 +425,8 @@ trace(){
echo "Usage:"
echo " kubectl ko trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]"
echo " kubectl ko trace {namespace/podname} {target ip address} [target mac address] arp {request|reply}"
echo " kubectl ko trace {node//nodename} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]"
echo " kubectl ko trace {node//nodename} {target ip address} [target mac address] arp {request|reply}"
exit 1
;;
esac
Expand All @@ -416,25 +437,25 @@ trace(){
echo ""
echo ""

inPort=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=ofport find interface external_id:iface-id="$podName"."$namespace")
local inPort=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=ofport find interface external_id:iface-id="$lsp")
case $type in
icmp)
set -x
kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac"
kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$srcIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac"
;;
tcp|udp)
set -x
kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac,${type}_src=1000,${type}_dst=$4"
kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$srcIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac,${type}_src=1000,${type}_dst=$4"
;;
arp)
case "$4" in
""|request)
set -x
kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,arp,arp_op=1,dl_src=$mac,dl_dst=$dstMac,arp_spa=$podIP,arp_tpa=$dst,arp_sha=$mac,arp_tha=00:00:00:00:00:00"
kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,arp,arp_op=1,dl_src=$mac,dl_dst=$dstMac,arp_spa=$srcIP,arp_tpa=$dst,arp_sha=$mac,arp_tha=00:00:00:00:00:00"
;;
reply)
set -x
kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,arp,arp_op=2,dl_src=$mac,dl_dst=$dstMac,arp_spa=$podIP,arp_tpa=$dst,arp_sha=$mac,arp_tha=$dstMac"
kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,arp,arp_op=2,dl_src=$mac,dl_dst=$dstMac,arp_spa=$srcIP,arp_tpa=$dst,arp_sha=$mac,arp_tha=$dstMac"
;;
esac
;;
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/framework/framework.go
View file
Expand Up @@ -142,9 +142,9 @@ func (f *Framework) VersionPriorTo(major, minor uint) bool {
return f.ClusterVersionMajor < major || (f.ClusterVersionMajor == major && f.ClusterVersionMinor < minor)
}

func (f *Framework) SkipVersionPriorTo(major, minor uint, message string) {
func (f *Framework) SkipVersionPriorTo(major, minor uint, reason string) {
if f.VersionPriorTo(major, minor) {
ginkgo.Skip(message)
ginkgo.Skip(reason)
}
}

Expand Down

0 comments on commit 53bfcf4

Please sign in to comment.