Skip to content

Commit

Permalink
underlay: fix link name exchange (#2516)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed Mar 22, 2023
1 parent e971095 commit 569b576
Show file tree
Hide file tree
Showing 6 changed files with 151 additions and 215 deletions.
147 changes: 14 additions & 133 deletions dist/images/start-ovs.sh
View file
Expand Up @@ -81,151 +81,32 @@ ovsdb_server_ctl="/var/run/openvswitch/ovsdb-server.$(cat /var/run/openvswitch/o
ovs-appctl -t "$ovsdb_server_ctl" vlog/set jsonrpc:file:err
ovs-appctl -t "$ovsdb_server_ctl" vlog/set reconnect:file:err

function exchange_link_names() {
mappings=($(ovs-vsctl --if-exists get open . external-ids:ovn-bridge-mappings | tr -d '"' | tr ',' ' '))
bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn external-ids:exchange-link-name=true))
function handle_underlay_bridges() {
bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn))
for br in ${bridges[@]}; do
provider=""
for m in ${mappings[*]}; do
if echo $m | grep -q ":$br"'$'; then
provider=${m%:$br}
break
fi
done
if [ "x$provider" = "x" ]; then
echo "error: failed to get provider name for bridge $br"
continue
if ! ip link show $br >/dev/null; then
# the bridge does not exist, leave it to be handled by kube-ovn-cni
echo "deleting ovs bridge $br"
ovs-vsctl --no-wait del-br $br
fi
done

port="br-$provider"
if ip link show $port 2>/dev/null; then
echo "link $port already exists"
continue
fi
if ! ip link show $br 2>/dev/null; then
echo "link $br does not exists"
continue
bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn external-ids:exchange-link-name=true))
for br in ${bridges[@]}; do
if [ -z $(ip link show $br type openvswitch 2>/dev/null || true) ]; then
# the bridge does not exist, leave it to be handled by kube-ovn-cni
echo "deleting ovs bridge $br"
ovs-vsctl --no-wait del-br $br
fi

echo "change link name from $br to $port"
ipv4_routes=($(ip -4 route show dev $br | tr ' ' '#'))
ipv6_routes=($(ip -6 route show dev $br | tr ' ' '#'))
ip link set $br down
ip link set $br name $port
ip link set $port up

# transfer IPv4 routes
default_ipv4_routes=()
for route in ${ipv4_routes[@]}; do
r=$(echo $route | tr '#' ' ')
if echo $r | grep -q -w 'scope link'; then
printf "add/replace IPv4 route $r to $port\n"
ip -4 route replace $r dev $port
else
default_ipv4_routes=(${default_ipv4_routes[@]} $route)
fi
done
for route in ${default_ipv4_routes[@]}; do
r=$(echo $route | tr '#' ' ')
printf "add/replace IPv4 route $r to $port\n"
ip -4 route replace $r dev $port
done

# transfer IPv6 routes
default_ipv6_routes=()
for route in ${ipv6_routes[@]}; do
r=$(echo $route | tr '#' ' ')
if echo $r | grep -q -w 'scope link'; then
printf "add/replace IPv6 route $r to $port\n"
ip -6 route replace $r dev $port
else
default_ipv6_routes=(${default_ipv6_routes[@]} $route)
fi
done
for route in ${default_ipv6_routes[@]}; do
r=$(echo $route | tr '#' ' ')
printf "add/replace IPv6 route $r to $port\n"
ip -6 route replace $r dev $port
done
done
}

exchange_link_names
handle_underlay_bridges

# Start vswitchd. restart will automatically set/unset flow-restore-wait which is not what we want
/usr/share/openvswitch/scripts/ovs-ctl restart --no-ovsdb-server --system-id=random --no-mlockall
/usr/share/openvswitch/scripts/ovs-ctl --protocol=udp --dport=6081 enable-protocol

sleep 1

function handle_underlay_bridges() {
bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn))
for br in ${bridges[@]}; do
echo "handle bridge $br"
ip link set $br up

ports=($(ovs-vsctl list-ports $br))
for port in ${ports[@]}; do
port_type=$(ovs-vsctl --no-heading --columns=type find interface name=$port)
if [ ! "x$port_type" = 'x""' ]; then
continue
fi

echo "handle port $port on bridge $br"
ipv4_routes=($(ip -4 route show dev $port | tr ' ' '#'))
ipv6_routes=($(ip -6 route show dev $port | tr ' ' '#'))

set +o pipefail
addresses=($(ip addr show dev $port | grep -E '^\s*inet[6]?\s+' | grep -w global | awk '{print $2}'))
set -o pipefail

# transfer IP addresses
for addr in ${addresses[@]}; do
printf "delete address $addr on $port\n"
ip addr del $addr dev $port || true
printf "add/replace address $addr to $br\n"
ip addr replace $addr dev $br
done

# transfer IPv4 routes
default_ipv4_routes=()
for route in ${ipv4_routes[@]}; do
r=$(echo $route | tr '#' ' ')
if echo $r | grep -q -w 'scope link'; then
printf "add/replace IPv4 route $r to $br\n"
ip -4 route replace $r dev $br
else
default_ipv4_routes=(${default_ipv4_routes[@]} $route)
fi
done
for route in ${default_ipv4_routes[@]}; do
r=$(echo $route | tr '#' ' ')
printf "add/replace IPv4 route $r to $br\n"
ip -4 route replace $r dev $br
done

# transfer IPv6 routes
default_ipv6_routes=()
for route in ${ipv6_routes[@]}; do
r=$(echo $route | tr '#' ' ')
if echo $r | grep -q -w 'scope link'; then
printf "add/replace IPv6 route $r to $br\n"
ip -6 route replace $r dev $br
else
default_ipv6_routes=(${default_ipv6_routes[@]} $route)
fi
done
for route in ${default_ipv6_routes[@]}; do
r=$(echo $route | tr '#' ' ')
printf "add/replace IPv6 route $r to $br\n"
ip -6 route replace $r dev $br
done
done
done
}

handle_underlay_bridges

function gen_conn_str {
if [[ -z "${OVN_DB_IPS}" ]]; then
if [[ "$ENABLE_SSL" == "false" ]]; then
Expand Down
4 changes: 4 additions & 0 deletions pkg/daemon/controller.go
View file
Expand Up @@ -323,6 +323,10 @@ func (c *Controller) recordProviderNetworkErr(providerNetwork string, errMsg str
break
}
}
if currentPod == nil {
klog.Warning("failed to get self pod")
return
}
} else {
if currentPod, err = c.config.KubeClient.CoreV1().Pods(c.localNamespace).Get(context.Background(), c.localPodName, metav1.GetOptions{}); err != nil {
klog.Errorf("failed to get pod %s, %v", c.localPodName, err)
Expand Down
37 changes: 13 additions & 24 deletions pkg/daemon/init.go
View file
Expand Up @@ -148,34 +148,11 @@ func ovsCleanProviderNetwork(provider string) error {
for idx, m = range brMappings {
if strings.HasPrefix(m, mappingPrefix) {
brName = m[len(mappingPrefix):]
klog.V(3).Infof("found bridge name for provider %s: %s", provider, brName)
break
}
}

if output, err = ovs.Exec("list-br"); err != nil {
return fmt.Errorf("failed to list OVS bridge %v: %q", err, output)
}

if !util.ContainsString(strings.Split(output, "\n"), brName) {
return nil
}

// get host nic
if output, err = ovs.Exec("list-ports", brName); err != nil {
return fmt.Errorf("failed to list ports of OVS bridge %s, %v: %q", brName, err, output)
}

// remove host nic from the external bridge
if output != "" {
for _, port := range strings.Split(output, "\n") {
if err = removeProviderNic(port, brName); err != nil {
errMsg := fmt.Errorf("failed to remove port %s from external bridge %s: %v", port, brName, err)
klog.Error(errMsg)
return errMsg
}
}
}

if idx != len(brMappings) {
brMappings = append(brMappings[:idx], brMappings[idx+1:]...)
if len(brMappings) == 0 {
Expand Down Expand Up @@ -207,6 +184,15 @@ func ovsCleanProviderNetwork(provider string) error {
return fmt.Errorf("failed to set ovn-chassis-mac-mappings, %v: %q", err, output)
}

if output, err = ovs.Exec("list-br"); err != nil {
return fmt.Errorf("failed to list OVS bridge %v: %q", err, output)
}

if !util.ContainsString(strings.Split(output, "\n"), brName) {
klog.V(3).Infof("ovs bridge %s not found", brName)
return nil
}

// get host nic
if output, err = ovs.Exec("list-ports", brName); err != nil {
return fmt.Errorf("failed to list ports of OVS bridge %s, %v: %q", brName, err, output)
Expand All @@ -215,11 +201,13 @@ func ovsCleanProviderNetwork(provider string) error {
// remove host nic from the external bridge
if output != "" {
for _, port := range strings.Split(output, "\n") {
klog.V(3).Infof("removing ovs port %s from bridge %s", port, brName)
if err = removeProviderNic(port, brName); err != nil {
errMsg := fmt.Errorf("failed to remove port %s from external bridge %s: %v", port, brName, err)
klog.Error(errMsg)
return errMsg
}
klog.V(3).Infof("ovs port %s has been removed from bridge %s", port, brName)
}
}

Expand All @@ -228,6 +216,7 @@ func ovsCleanProviderNetwork(provider string) error {
if output, err = ovs.Exec(ovs.IfExists, "del-br", brName); err != nil {
return fmt.Errorf("failed to remove OVS bridge %s, %v: %q", brName, err, output)
}
klog.V(3).Infof("ovs bridge %s has been deleted", brName)

if br := util.ExternalBridgeName(provider); br != brName {
if _, err = changeProvideNicName(br, brName); err != nil {
Expand Down
53 changes: 46 additions & 7 deletions pkg/daemon/init_linux.go
View file
@@ -1,7 +1,7 @@
package daemon

import (
"syscall"
"time"

"k8s.io/klog/v2"

Expand Down Expand Up @@ -45,6 +45,39 @@ func nmSetManaged(device string, managed bool) error {
return nil
}

// wait systemd-networkd to finish interface configuration
func waitNetworkdConfiguration(linkIndex int) {
done := make(chan struct{})
ch := make(chan netlink.RouteUpdate)
if err := netlink.RouteSubscribe(ch, done); err != nil {
klog.Warningf("failed to subscribe route update events: %v", err)
klog.Info("Waiting 100ms ...")
time.Sleep(100 * time.Millisecond)
return
}

// wait route event on the link for 50ms
timer := time.NewTimer(50 * time.Millisecond)
for {
select {
case <-timer.C:
// timeout, interface configuration is expected to be completed
done <- struct{}{}
return
case event := <-ch:
if event.LinkIndex == linkIndex {
// received a route event on the link
// stop the timer
if !timer.Stop() {
<-timer.C
}
// reset the timer, wait for another 50ms
timer.Reset(50 * time.Millisecond)
}
}
}
}

func changeProvideNicName(current, target string) (bool, error) {
link, err := netlink.LinkByName(current)
if err != nil {
Expand All @@ -56,17 +89,17 @@ func changeProvideNicName(current, target string) (bool, error) {
return false, err
}
if link.Type() == "openvswitch" {
klog.Infof("%s is an openvswitch interface, skip", current)
klog.V(3).Infof("%s is an openvswitch interface, skip", current)
return true, nil
}

// set link unmanaged by NetworkManager to avoid getting new IP by DHCP
// set link unmanaged by NetworkManager
if err = nmSetManaged(current, false); err != nil {
klog.Errorf("failed set device %s to unmanaged by NetworkManager: %v", current, err)
return false, err
}

klog.Infof("change nic name from %s to %s", current, target)
klog.Infof("renaming link %s as %s", current, target)
addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL)
if err != nil {
klog.Errorf("failed to list addresses of link %s: %v", current, err)
Expand All @@ -90,15 +123,20 @@ func changeProvideNicName(current, target string) (bool, error) {
klog.Errorf("failed to set link %s up: %v", target, err)
return false, err
}
klog.Infof("link %s has been renamed as %s", current, target)

waitNetworkdConfiguration(link.Attrs().Index)

for _, addr := range addresses {
if addr.IP.IsLinkLocalUnicast() {
continue
}
addr.Label = ""
if err = netlink.AddrReplace(link, &addr); err != nil {
klog.Errorf("failed to replace address %s: %v", addr.String(), err)
klog.Errorf("failed to replace address %q: %v", addr.String(), err)
return false, err
}
klog.Infof("address %q has been added/replaced to link %s", addr.String(), target)
}

for _, scope := range routeScopeOrders {
Expand All @@ -107,10 +145,11 @@ func changeProvideNicName(current, target string) (bool, error) {
continue
}
if route.Scope == scope {
if err = netlink.RouteReplace(&route); err != nil && err != syscall.EEXIST {
klog.Errorf("failed to replace route %s: %v", route.String(), err)
if err = netlink.RouteReplace(&route); err != nil {
klog.Errorf("failed to replace route %q to %s: %v", route.String(), target, err)
return false, err
}
klog.Infof("route %q has been added/replaced to link %s", route.String(), target)
}
}
}
Expand Down

0 comments on commit 569b576

Please sign in to comment.