Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

underlay: fix link name exchange #2516

Merged
merged 6 commits into from
Mar 22, 2023
Merged

underlay: fix link name exchange #2516

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
674e57e
underlay: fix link name exchange
zhangzujian Mar 19, 2023
23b9e37
split systemd-networkd configuration waiting
zhangzujian Mar 20, 2023
8a2ca49
wait networkd in ovs startup script
zhangzujian Mar 20, 2023
1102744
some fixes
zhangzujian Mar 20, 2023
30257e5
fix link name exchange on os startup
zhangzujian Mar 20, 2023
9193f52
minor fixes
zhangzujian Mar 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
147 changes: 14 additions & 133 deletions dist/images/start-ovs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,151 +81,32 @@ ovsdb_server_ctl="/var/run/openvswitch/ovsdb-server.$(cat /var/run/openvswitch/o
ovs-appctl -t "$ovsdb_server_ctl" vlog/set jsonrpc:file:err
ovs-appctl -t "$ovsdb_server_ctl" vlog/set reconnect:file:err

function exchange_link_names() {
mappings=($(ovs-vsctl --if-exists get open . external-ids:ovn-bridge-mappings | tr -d '"' | tr ',' ' '))
bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn external-ids:exchange-link-name=true))
function handle_underlay_bridges() {
bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn))
for br in ${bridges[@]}; do
provider=""
for m in ${mappings[*]}; do
if echo $m | grep -q ":$br"'$'; then
provider=${m%:$br}
break
fi
done
if [ "x$provider" = "x" ]; then
echo "error: failed to get provider name for bridge $br"
continue
if ! ip link show $br >/dev/null; then
# the bridge does not exist, leave it to be handled by kube-ovn-cni
echo "deleting ovs bridge $br"
ovs-vsctl --no-wait del-br $br
fi
done

port="br-$provider"
if ip link show $port 2>/dev/null; then
echo "link $port already exists"
continue
fi
if ! ip link show $br 2>/dev/null; then
echo "link $br does not exists"
continue
bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn external-ids:exchange-link-name=true))
for br in ${bridges[@]}; do
if [ -z $(ip link show $br type openvswitch 2>/dev/null || true) ]; then
# the bridge does not exist, leave it to be handled by kube-ovn-cni
echo "deleting ovs bridge $br"
ovs-vsctl --no-wait del-br $br
fi

echo "change link name from $br to $port"
ipv4_routes=($(ip -4 route show dev $br | tr ' ' '#'))
ipv6_routes=($(ip -6 route show dev $br | tr ' ' '#'))
ip link set $br down
ip link set $br name $port
ip link set $port up

# transfer IPv4 routes
default_ipv4_routes=()
for route in ${ipv4_routes[@]}; do
r=$(echo $route | tr '#' ' ')
if echo $r | grep -q -w 'scope link'; then
printf "add/replace IPv4 route $r to $port\n"
ip -4 route replace $r dev $port
else
default_ipv4_routes=(${default_ipv4_routes[@]} $route)
fi
done
for route in ${default_ipv4_routes[@]}; do
r=$(echo $route | tr '#' ' ')
printf "add/replace IPv4 route $r to $port\n"
ip -4 route replace $r dev $port
done

# transfer IPv6 routes
default_ipv6_routes=()
for route in ${ipv6_routes[@]}; do
r=$(echo $route | tr '#' ' ')
if echo $r | grep -q -w 'scope link'; then
printf "add/replace IPv6 route $r to $port\n"
ip -6 route replace $r dev $port
else
default_ipv6_routes=(${default_ipv6_routes[@]} $route)
fi
done
for route in ${default_ipv6_routes[@]}; do
r=$(echo $route | tr '#' ' ')
printf "add/replace IPv6 route $r to $port\n"
ip -6 route replace $r dev $port
done
done
}

exchange_link_names
handle_underlay_bridges

# Start vswitchd. restart will automatically set/unset flow-restore-wait which is not what we want
/usr/share/openvswitch/scripts/ovs-ctl restart --no-ovsdb-server --system-id=random --no-mlockall
/usr/share/openvswitch/scripts/ovs-ctl --protocol=udp --dport=6081 enable-protocol

sleep 1

function handle_underlay_bridges() {
bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn))
for br in ${bridges[@]}; do
echo "handle bridge $br"
ip link set $br up

ports=($(ovs-vsctl list-ports $br))
for port in ${ports[@]}; do
port_type=$(ovs-vsctl --no-heading --columns=type find interface name=$port)
if [ ! "x$port_type" = 'x""' ]; then
continue
fi

echo "handle port $port on bridge $br"
ipv4_routes=($(ip -4 route show dev $port | tr ' ' '#'))
ipv6_routes=($(ip -6 route show dev $port | tr ' ' '#'))

set +o pipefail
addresses=($(ip addr show dev $port | grep -E '^\s*inet[6]?\s+' | grep -w global | awk '{print $2}'))
set -o pipefail

# transfer IP addresses
for addr in ${addresses[@]}; do
printf "delete address $addr on $port\n"
ip addr del $addr dev $port || true
printf "add/replace address $addr to $br\n"
ip addr replace $addr dev $br
done

# transfer IPv4 routes
default_ipv4_routes=()
for route in ${ipv4_routes[@]}; do
r=$(echo $route | tr '#' ' ')
if echo $r | grep -q -w 'scope link'; then
printf "add/replace IPv4 route $r to $br\n"
ip -4 route replace $r dev $br
else
default_ipv4_routes=(${default_ipv4_routes[@]} $route)
fi
done
for route in ${default_ipv4_routes[@]}; do
r=$(echo $route | tr '#' ' ')
printf "add/replace IPv4 route $r to $br\n"
ip -4 route replace $r dev $br
done

# transfer IPv6 routes
default_ipv6_routes=()
for route in ${ipv6_routes[@]}; do
r=$(echo $route | tr '#' ' ')
if echo $r | grep -q -w 'scope link'; then
printf "add/replace IPv6 route $r to $br\n"
ip -6 route replace $r dev $br
else
default_ipv6_routes=(${default_ipv6_routes[@]} $route)
fi
done
for route in ${default_ipv6_routes[@]}; do
r=$(echo $route | tr '#' ' ')
printf "add/replace IPv6 route $r to $br\n"
ip -6 route replace $r dev $br
done
done
done
}

handle_underlay_bridges

function gen_conn_str {
if [[ -z "${OVN_DB_IPS}" ]]; then
if [[ "$ENABLE_SSL" == "false" ]]; then
Expand Down
4 changes: 4 additions & 0 deletions pkg/daemon/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,10 @@ func (c *Controller) recordProviderNetworkErr(providerNetwork string, errMsg str
break
}
}
if currentPod == nil {
klog.Warning("failed to get self pod")
return
}
} else {
if currentPod, err = c.config.KubeClient.CoreV1().Pods(c.localNamespace).Get(context.Background(), c.localPodName, metav1.GetOptions{}); err != nil {
klog.Errorf("failed to get pod %s, %v", c.localPodName, err)
Expand Down
37 changes: 13 additions & 24 deletions pkg/daemon/init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,34 +148,11 @@ func ovsCleanProviderNetwork(provider string) error {
for idx, m = range brMappings {
if strings.HasPrefix(m, mappingPrefix) {
brName = m[len(mappingPrefix):]
klog.V(3).Infof("found bridge name for provider %s: %s", provider, brName)
break
}
}

if output, err = ovs.Exec("list-br"); err != nil {
return fmt.Errorf("failed to list OVS bridge %v: %q", err, output)
}

if !util.ContainsString(strings.Split(output, "\n"), brName) {
return nil
}

// get host nic
if output, err = ovs.Exec("list-ports", brName); err != nil {
return fmt.Errorf("failed to list ports of OVS bridge %s, %v: %q", brName, err, output)
}

// remove host nic from the external bridge
if output != "" {
for _, port := range strings.Split(output, "\n") {
if err = removeProviderNic(port, brName); err != nil {
errMsg := fmt.Errorf("failed to remove port %s from external bridge %s: %v", port, brName, err)
klog.Error(errMsg)
return errMsg
}
}
}

if idx != len(brMappings) {
brMappings = append(brMappings[:idx], brMappings[idx+1:]...)
if len(brMappings) == 0 {
Expand Down Expand Up @@ -207,6 +184,15 @@ func ovsCleanProviderNetwork(provider string) error {
return fmt.Errorf("failed to set ovn-chassis-mac-mappings, %v: %q", err, output)
}

if output, err = ovs.Exec("list-br"); err != nil {
return fmt.Errorf("failed to list OVS bridge %v: %q", err, output)
}

if !util.ContainsString(strings.Split(output, "\n"), brName) {
klog.V(3).Infof("ovs bridge %s not found", brName)
return nil
}

// get host nic
if output, err = ovs.Exec("list-ports", brName); err != nil {
return fmt.Errorf("failed to list ports of OVS bridge %s, %v: %q", brName, err, output)
Expand All @@ -215,11 +201,13 @@ func ovsCleanProviderNetwork(provider string) error {
// remove host nic from the external bridge
if output != "" {
for _, port := range strings.Split(output, "\n") {
klog.V(3).Infof("removing ovs port %s from bridge %s", port, brName)
if err = removeProviderNic(port, brName); err != nil {
errMsg := fmt.Errorf("failed to remove port %s from external bridge %s: %v", port, brName, err)
klog.Error(errMsg)
return errMsg
}
klog.V(3).Infof("ovs port %s has been removed from bridge %s", port, brName)
}
}

Expand All @@ -228,6 +216,7 @@ func ovsCleanProviderNetwork(provider string) error {
if output, err = ovs.Exec(ovs.IfExists, "del-br", brName); err != nil {
return fmt.Errorf("failed to remove OVS bridge %s, %v: %q", brName, err, output)
}
klog.V(3).Infof("ovs bridge %s has been deleted", brName)

if br := util.ExternalBridgeName(provider); br != brName {
if _, err = changeProvideNicName(br, brName); err != nil {
Expand Down
53 changes: 46 additions & 7 deletions pkg/daemon/init_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package daemon

import (
"syscall"
"time"

"k8s.io/klog/v2"

Expand Down Expand Up @@ -45,6 +45,39 @@ func nmSetManaged(device string, managed bool) error {
return nil
}

// wait systemd-networkd to finish interface configuration
func waitNetworkdConfiguration(linkIndex int) {
done := make(chan struct{})
ch := make(chan netlink.RouteUpdate)
if err := netlink.RouteSubscribe(ch, done); err != nil {
klog.Warningf("failed to subscribe route update events: %v", err)
klog.Info("Waiting 100ms ...")
time.Sleep(100 * time.Millisecond)
return
}

// wait route event on the link for 50ms
timer := time.NewTimer(50 * time.Millisecond)
for {
select {
case <-timer.C:
// timeout, interface configuration is expected to be completed
done <- struct{}{}
return
case event := <-ch:
if event.LinkIndex == linkIndex {
// received a route event on the link
// stop the timer
if !timer.Stop() {
<-timer.C
}
// reset the timer, wait for another 50ms
timer.Reset(50 * time.Millisecond)
}
}
}
}

func changeProvideNicName(current, target string) (bool, error) {
link, err := netlink.LinkByName(current)
if err != nil {
Expand All @@ -56,17 +89,17 @@ func changeProvideNicName(current, target string) (bool, error) {
return false, err
}
if link.Type() == "openvswitch" {
klog.Infof("%s is an openvswitch interface, skip", current)
klog.V(3).Infof("%s is an openvswitch interface, skip", current)
return true, nil
}

// set link unmanaged by NetworkManager to avoid getting new IP by DHCP
// set link unmanaged by NetworkManager
if err = nmSetManaged(current, false); err != nil {
klog.Errorf("failed set device %s to unmanaged by NetworkManager: %v", current, err)
return false, err
}

klog.Infof("change nic name from %s to %s", current, target)
klog.Infof("renaming link %s as %s", current, target)
addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL)
if err != nil {
klog.Errorf("failed to list addresses of link %s: %v", current, err)
Expand All @@ -90,15 +123,20 @@ func changeProvideNicName(current, target string) (bool, error) {
klog.Errorf("failed to set link %s up: %v", target, err)
return false, err
}
klog.Infof("link %s has been renamed as %s", current, target)

waitNetworkdConfiguration(link.Attrs().Index)

for _, addr := range addresses {
if addr.IP.IsLinkLocalUnicast() {
continue
}
addr.Label = ""
if err = netlink.AddrReplace(link, &addr); err != nil {
klog.Errorf("failed to replace address %s: %v", addr.String(), err)
klog.Errorf("failed to replace address %q: %v", addr.String(), err)
return false, err
}
klog.Infof("address %q has been added/replaced to link %s", addr.String(), target)
}

for _, scope := range routeScopeOrders {
Expand All @@ -107,10 +145,11 @@ func changeProvideNicName(current, target string) (bool, error) {
continue
}
if route.Scope == scope {
if err = netlink.RouteReplace(&route); err != nil && err != syscall.EEXIST {
klog.Errorf("failed to replace route %s: %v", route.String(), err)
if err = netlink.RouteReplace(&route); err != nil {
klog.Errorf("failed to replace route %q to %s: %v", route.String(), target, err)
return false, err
}
klog.Infof("route %q has been added/replaced to link %s", route.String(), target)
}
}
}
Expand Down