Skip to content

Commit

Permalink
kubectl-ko: fix destination mac (#1801)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed Aug 15, 2022
1 parent e8816e9 commit 8b2df58
Showing 1 changed file with 179 additions and 68 deletions.
247 changes: 179 additions & 68 deletions dist/images/kubectl-ko
View file
Expand Up @@ -25,6 +25,105 @@ showHelp(){
echo " reload restart all kube-ovn components"
}

# usage: ipv4_to_hex 192.168.0.1
ipv4_to_hex(){
printf "%02x" ${1//./ }
}

# convert hex to dec (portable version)
hex2dec(){
for i in $(echo "$@"); do
printf "%d\n" "$(( 0x$i ))"
done
}

# https://github.com/chmduquesne/wg-ip
# usage: expand_ipv6 2001::1
expand_ipv6(){
local ip=$1

# prepend 0 if we start with :
echo $ip | grep -qs "^:" && ip="0${ip}"

# expand ::
if echo $ip | grep -qs "::"; then
local colons=$(echo $ip | sed 's/[^:]//g')
local missing=$(echo ":::::::::" | sed "s/$colons//")
local expanded=$(echo $missing | sed 's/:/:0/g')
ip=$(echo $ip | sed "s/::/$expanded/")
fi

local blocks=$(echo $ip | grep -o "[0-9a-f]\+")
set $blocks

printf "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n" \
$(hex2dec $@)
}

# convert an IPv6 address to bytes
ipv6_bytes(){
for x in $(expand_ipv6 $1 | tr ':' ' '); do
printf "%d %d " $((0x$x >> 8 & 0xff)) $((0x$x & 0xff))
done
echo
}

# usage: ipIsInCidr 192.168.0.1 192.168.0.0/24
# return: 0 for true, 1 for false
ipIsInCidr(){
local ip=$1
local cidr=$2

if [[ $ip =~ .*:.* ]]; then
# IPv6
cidr=${cidr#*,}
local network=${cidr%/*}
local prefix=${cidr#*/}
local ip_bytes=($(ipv6_bytes $ip))
local network_bytes=($(ipv6_bytes $network))
for ((i=0; i<${#ip_bytes[*]}; i++)); do
if [ ${ip_bytes[$i]} -eq ${network_bytes[$i]} ]; then
continue
fi

if [ $((($i+1)*8)) -le $prefix ]; then
return 1
fi
if [ $(($i*8)) -ge $prefix ]; then
return 0
fi
if [ $((($i+1)*8)) -le $prefix ]; then
return 1
fi

local bits=$(($prefix-$i*8))
local mask=$((0xff<<$bits & 0xff))
# TODO: check whether the IP is network/broadcast address
if [ $((${ip_bytes[$i]} & $mask)) -ne ${network_bytes[$i]} ]; then
return 1
fi
done

return 0
fi

# IPv4
cidr=${cidr%,*}
local network=${cidr%/*}
local prefix=${cidr#*/}
local ip_hex=$(ipv4_to_hex $ip)
local ip_dec=$((0x$ip_hex))
local network_hex=$(ipv4_to_hex $network)
local network_dec=$((0x$network_hex))
local broadcast_dec=$(($network_dec + 2**$prefix - 1))
# TODO: check whether the IP is network/broadcast address
if [ $ip_dec -gt $network_dec -a $ip_dec -lt $broadcast_dec ]; then
return 0
fi

return 1
}

tcpdump(){
namespacedPod="$1"; shift
namespace=$(echo "$namespacedPod" | cut -d "/" -f1)
Expand Down Expand Up @@ -69,9 +168,9 @@ tcpdump(){

trace(){
namespacedPod="$1"
namespace=$(echo "$1" | cut -d "/" -f1)
podName=$(echo "$1" | cut -d "/" -f2)
if [ "$podName" = "$1" ]; then
namespace=$(echo "$namespacedPod" | cut -d "/" -f1)
podName=$(echo "$namespacedPod" | cut -d "/" -f2)
if [ "$podName" = "$namespacedPod" ]; then
namespace="default"
fi

Expand All @@ -81,6 +180,12 @@ trace(){
exit 1
fi

hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
if [ "$hostNetwork" = "true" ]; then
echo "Can not trace host network pod"
exit 1
fi

af="4"
nw="nw"
proto=""
Expand All @@ -90,18 +195,6 @@ trace(){
proto="6"
fi

hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
if [ "$hostNetwork" = "true" ]; then
echo "Can not trace host network pod"
exit 1
fi

ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
if [ -z "$ls" ]; then
echo "pod address not ready"
exit 1
fi

podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath="{.status.podIPs[*].ip}"))
if [ ${#podIPs[@]} -eq 0 ]; then
podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address} | sed 's/,/ /g'))
Expand All @@ -111,9 +204,6 @@ trace(){
fi
fi

mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})

podIP=""
for ip in ${podIPs[@]}; do
if [ "$af" = "4" ]; then
Expand All @@ -128,76 +218,97 @@ trace(){
done

if [ -z "$podIP" ]; then
echo "Pod has no IPv$af address"
echo "Pod $namespacedPod has no IPv$af address"
exit 1
fi

gwMac=""
vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})
logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway})
if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then
gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway})
if [[ "$gateway" =~ .*,.* ]]; then
if [ "$af" = "4" ]; then
gateway=${gateway%%,*}
else
gateway=${gateway##*,}
fi
fi
ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
if [ -z "$ls" ]; then
echo "pod address not ready"
exit 1
fi

ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep -w kube-ovn-cni | grep " $nodeName " | awk '{print $1}')
if [ -z "$ovnCni" ]; then
echo "No kube-ovn-cni Pod running on node $nodeName"
exit 1
fi
local cidr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/cidr})
mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})

nicName=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r')
if [ -z "$nicName" ]; then
echo "nic doesn't exist on node $nodeName"
exit 1
dstMac=""
if ipIsInCidr $dst $cidr; then
set +o pipefail
if [ $af -eq 4 ]; then
dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -w "$(echo $dst | tr . '\.')" | awk '{print $1}')
else
dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -i " $dst\$" | awk '{print $1}')
fi
set -o pipefail
fi
if [ -z "$dstMac" ]; then
vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})
logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway})
if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then
gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway})
if [[ "$gateway" =~ .*,.* ]]; then
if [ "$af" = "4" ]; then
gateway=${gateway%%,*}
else
gateway=${gateway##*,}
fi
fi

podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type})
podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//')
if [ "$podNicType" != "internal-port" ]; then
nicName="eth0"
fi
ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep -w kube-ovn-cni | grep " $nodeName " | awk '{print $1}')
if [ -z "$ovnCni" ]; then
echo "No kube-ovn-cni Pod running on node $nodeName"
exit 1
fi

if [[ "$gateway" =~ .*:.* ]]; then
cmd="ndisc6 -q $gateway $nicName"
output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName")
else
cmd="arping -c3 -C1 -i1 -I $nicName $gateway"
output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway")
fi
nicName=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r')
if [ -z "$nicName" ]; then
echo "nic doesn't exist on node $nodeName"
exit 1
fi

if [ $? -ne 0 ]; then
echo "failed to run '$cmd' in Pod's netns"
exit 1
fi
gwMac=$(echo "$output" | grep -o -E '([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}')
else
lr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router})
if [ -z "$lr" ]; then
lr=$(kubectl get subnet "$ls" -o jsonpath={.spec.vpc})
podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type})
podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//')
if [ "$podNicType" != "internal-port" ]; then
nicName="eth0"
fi

if [[ "$gateway" =~ .*:.* ]]; then
cmd="ndisc6 -q $gateway $nicName"
output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName")
else
cmd="arping -c3 -C1 -i1 -I $nicName $gateway"
output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway")
fi

if [ $? -ne 0 ]; then
echo "failed to run '$cmd' in Pod's netns"
exit 1
fi
dstMac=$(echo "$output" | grep -o -E '([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}')
else
lr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router})
if [ -z "$lr" ]; then
lr=$(kubectl get subnet "$ls" -o jsonpath={.spec.vpc})
fi
dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name="$lr"-"$ls" | tr -d '\r')
fi
gwMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name="$lr"-"$ls" | tr -d '\r')
fi

if [ -z "$gwMac" ]; then
echo "get gw mac failed"
if [ -z "$dstMac" ]; then
echo "failed to get destination mac"
exit 1
fi

type="$3"
case $type in
icmp)
set -x
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst"
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst"
;;
tcp|udp)
set -x
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4"
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4"
;;
*)
echo "type $type not supported"
Expand All @@ -222,11 +333,11 @@ trace(){
case $type in
icmp)
set -x
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac"
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac"
;;
tcp|udp)
set -x
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac,${type}_src=1000,${type}_dst=$4"
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac,${type}_src=1000,${type}_dst=$4"
;;
*)
echo "type $type not supported"
Expand Down

0 comments on commit 8b2df58

Please sign in to comment.