From d7a9f5e91c5a44deb9801e4538386512e45da627 Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Fri, 5 Aug 2022 11:47:36 +0800 Subject: [PATCH] feature: support exchange link names of OVS bridge and provider nic in underlay networks (#1736) --- dist/images/install.sh | 2 + dist/images/start-ovs.sh | 71 +++++++++++++ go.mod | 2 + go.sum | 5 + pkg/apis/kubeovn/v1/types.go | 1 + pkg/controller/config.go | 37 +++---- pkg/controller/init.go | 1 + pkg/daemon/controller.go | 2 +- pkg/daemon/init.go | 187 ++++++++++++++++++++++++++++++---- pkg/daemon/ovs.go | 14 +-- test/e2e/e2e_suite_test.go | 3 +- test/e2e/underlay/underlay.go | 20 +++- yamls/crd.yaml | 2 + 13 files changed, 294 insertions(+), 53 deletions(-) diff --git a/dist/images/install.sh b/dist/images/install.sh index 5893b47716a..99febbfecc0 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -702,6 +702,8 @@ spec: type: array items: type: string + exchangeLinkName: + type: boolean excludeNodes: type: array items: diff --git a/dist/images/start-ovs.sh b/dist/images/start-ovs.sh index cf0d03d0f60..6d8c206ff59 100755 --- a/dist/images/start-ovs.sh +++ b/dist/images/start-ovs.sh @@ -63,6 +63,77 @@ else ovs-vsctl --no-wait set open_vswitch . other_config:hw-offload=false fi +function exchange_link_names() { + mappings=($(ovs-vsctl --if-exists get open . external-ids:ovn-bridge-mappings | tr -d '"' | tr ',' ' ')) + bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn external-ids:exchange-link-name=true)) + for br in ${bridges[@]}; do + provider="" + for m in ${mappings[*]}; do + if echo $m | grep -q ":$br"'$'; then + provider=${m%:$br} + break + fi + done + if [ "x$provider" = "x" ]; then + echo "error: failed to get provider name for bridge $br" + continue + fi + + port="br-$provider" + if ip link show $port 2>/dev/null; then + echo "link $port already exists" + continue + fi + if ! ip link show $br 2>/dev/null; then + echo "link $br does not exists" + continue + fi + + echo "change link name from $br to $port" + ipv4_routes=($(ip -4 route show dev $br | tr ' ' '#')) + ipv6_routes=($(ip -6 route show dev $br | tr ' ' '#')) + ip link set $br down + ip link set $br name $port + ip link set $port up + + # transfer IPv4 routes + default_ipv4_routes=() + for route in ${ipv4_routes[@]}; do + r=$(echo $route | tr '#' ' ') + if echo $r | grep -q -w 'scope link'; then + printf "add/replace IPv4 route $r to $port\n" + ip -4 route replace $r dev $port + else + default_ipv4_routes=(${default_ipv4_routes[@]} $route) + fi + done + for route in ${default_ipv4_routes[@]}; do + r=$(echo $route | tr '#' ' ') + printf "add/replace IPv4 route $r to $port\n" + ip -4 route replace $r dev $port + done + + # transfer IPv6 routes + default_ipv6_routes=() + for route in ${ipv6_routes[@]}; do + r=$(echo $route | tr '#' ' ') + if echo $r | grep -q -w 'scope link'; then + printf "add/replace IPv6 route $r to $port\n" + ip -6 route replace $r dev $port + else + default_ipv6_routes=(${default_ipv6_routes[@]} $route) + fi + done + for route in ${default_ipv6_routes[@]}; do + r=$(echo $route | tr '#' ' ') + printf "add/replace IPv6 route $r to $port\n" + ip -6 route replace $r dev $port + done + done +} + +exchange_link_names + # Start vswitchd. restart will automatically set/unset flow-restore-wait which is not what we want /usr/share/openvswitch/scripts/ovs-ctl start --no-ovsdb-server --system-id=random /usr/share/openvswitch/scripts/ovs-ctl --protocol=udp --dport=6081 enable-protocol diff --git a/go.mod b/go.mod index e6d01f7b5be..50cd7f93912 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.17 require ( github.com/Mellanox/sriovnet v1.0.3 + github.com/Wifx/gonetworkmanager v0.4.0 github.com/alauda/felix v3.6.6-0.20201207121355-187332daf314+incompatible github.com/cenkalti/backoff/v4 v4.1.3 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 @@ -59,6 +60,7 @@ require ( github.com/go-kit/kit v0.9.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.2.3 // indirect + github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect diff --git a/go.sum b/go.sum index 0ab243b3928..adabf516294 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/Wifx/gonetworkmanager v0.4.0 h1:iZ7o3z3YIqEEAa+bBOCwQkLLUQuxG02CdiE7NCe5y6A= +github.com/Wifx/gonetworkmanager v0.4.0/go.mod h1:EdhHf2O00IZXfMv9LC6CS6SgTwcMTg/ZSDhGvch0cs8= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28 h1:FW5M3SAwSGBdtTboeV5sI7kEY6zraApSZQxTUfZ7LQY= github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28/go.mod h1:dXpg+IAC2yp2IZQlEVmnmEc1rqEmSZzgNfu6+ai38J4= @@ -460,8 +462,11 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= +github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c h1:RBUpb2b14UnmRHNd2uHz20ZHLDK+SW5Us/vWF5IHRaY= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus/v5 v5.0.2/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= diff --git a/pkg/apis/kubeovn/v1/types.go b/pkg/apis/kubeovn/v1/types.go index 2b14a2d68f7..df850a2e243 100644 --- a/pkg/apis/kubeovn/v1/types.go +++ b/pkg/apis/kubeovn/v1/types.go @@ -260,6 +260,7 @@ type ProviderNetworkSpec struct { DefaultInterface string `json:"defaultInterface,omitempty"` CustomInterfaces []CustomInterface `json:"customInterfaces,omitempty"` ExcludeNodes []string `json:"excludeNodes,omitempty"` + ExchangeLinkName bool `json:"exchangeLinkName,omitempty"` } type ProviderNetworkStatus struct { diff --git a/pkg/controller/config.go b/pkg/controller/config.go index e91853698e3..7b19400ecf4 100644 --- a/pkg/controller/config.go +++ b/pkg/controller/config.go @@ -64,11 +64,12 @@ type Configuration struct { EnablePprof bool NodePgProbeTime int - NetworkType string - DefaultProviderName string - DefaultHostInterface string - DefaultVlanName string - DefaultVlanID int + NetworkType string + DefaultProviderName string + DefaultHostInterface string + DefaultExchangeLinkName bool + DefaultVlanName string + DefaultVlanID int EnableLb bool EnableNP bool @@ -115,18 +116,19 @@ func ParseFlags() (*Configuration, error) { argPprofPort = pflag.Int("pprof-port", 10660, "The port to get profiling data") argNodePgProbeTime = pflag.Int("nodepg-probe-time", 1, "The probe interval for node port-group, the unit is minute") - argNetworkType = pflag.String("network-type", util.NetworkTypeGeneve, "The ovn network type") - argDefaultProviderName = pflag.String("default-provider-name", "provider", "The vlan or vxlan type default provider interface name") - argDefaultInterfaceName = pflag.String("default-interface-name", "", "The default host interface name in the vlan/vxlan type") - argDefaultVlanName = pflag.String("default-vlan-name", "ovn-vlan", "The default vlan name") - argDefaultVlanID = pflag.Int("default-vlan-id", 1, "The default vlan id") - argPodNicType = pflag.String("pod-nic-type", "veth-pair", "The default pod network nic implementation type") - argEnableLb = pflag.Bool("enable-lb", true, "Enable load balancer") - argEnableNP = pflag.Bool("enable-np", true, "Enable network policy support") - argEnableEipSnat = pflag.Bool("enable-eip-snat", true, "Enable EIP and SNAT") - argEnableExternalVpc = pflag.Bool("enable-external-vpc", true, "Enable external vpc support") - argEnableEcmp = pflag.Bool("enable-ecmp", false, "Enable ecmp route for centralized subnet") - argKeepVmIP = pflag.Bool("keep-vm-ip", false, "Whether to keep ip for kubevirt pod when pod is rebuild") + argNetworkType = pflag.String("network-type", util.NetworkTypeGeneve, "The ovn network type") + argDefaultProviderName = pflag.String("default-provider-name", "provider", "The vlan or vxlan type default provider interface name") + argDefaultInterfaceName = pflag.String("default-interface-name", "", "The default host interface name in the vlan/vxlan type") + argDefaultExchangeLinkName = pflag.Bool("default-exchange-link-name", false, "exchange link names of OVS bridge and the provider nic in the default provider-network") + argDefaultVlanName = pflag.String("default-vlan-name", "ovn-vlan", "The default vlan name") + argDefaultVlanID = pflag.Int("default-vlan-id", 1, "The default vlan id") + argPodNicType = pflag.String("pod-nic-type", "veth-pair", "The default pod network nic implementation type") + argEnableLb = pflag.Bool("enable-lb", true, "Enable load balancer") + argEnableNP = pflag.Bool("enable-np", true, "Enable network policy support") + argEnableEipSnat = pflag.Bool("enable-eip-snat", true, "Enable EIP and SNAT") + argEnableExternalVpc = pflag.Bool("enable-external-vpc", true, "Enable external vpc support") + argEnableEcmp = pflag.Bool("enable-ecmp", false, "Enable ecmp route for centralized subnet") + argKeepVmIP = pflag.Bool("keep-vm-ip", false, "Whether to keep ip for kubevirt pod when pod is rebuild") argExternalGatewayConfigNS = pflag.String("external-gateway-config-ns", "kube-system", "The namespace of configmap external-gateway-config, default: kube-system") argExternalGatewayNet = pflag.String("external-gateway-net", "external", "The namespace of configmap external-gateway-config, default: external") @@ -178,6 +180,7 @@ func ParseFlags() (*Configuration, error) { DefaultVlanID: *argDefaultVlanID, DefaultProviderName: *argDefaultProviderName, DefaultHostInterface: *argDefaultInterfaceName, + DefaultExchangeLinkName: *argDefaultExchangeLinkName, DefaultVlanName: *argDefaultVlanName, PodName: os.Getenv("POD_NAME"), PodNamespace: os.Getenv("KUBE_NAMESPACE"), diff --git a/pkg/controller/init.go b/pkg/controller/init.go index 8c671635552..e1db9e75e30 100644 --- a/pkg/controller/init.go +++ b/pkg/controller/init.go @@ -435,6 +435,7 @@ func (c *Controller) initDefaultProviderNetwork() error { }, Spec: kubeovnv1.ProviderNetworkSpec{ DefaultInterface: c.config.DefaultHostInterface, + ExchangeLinkName: c.config.DefaultExchangeLinkName, }, } diff --git a/pkg/daemon/controller.go b/pkg/daemon/controller.go index e03104f927f..8a36fb452b1 100644 --- a/pkg/daemon/controller.go +++ b/pkg/daemon/controller.go @@ -291,7 +291,7 @@ func (c *Controller) initProviderNetwork(pn *kubeovnv1.ProviderNetwork, node *v1 var mtu int var err error - if mtu, err = ovsInitProviderNetwork(pn.Name, nic); err != nil { + if mtu, err = ovsInitProviderNetwork(pn.Name, nic, pn.Spec.ExchangeLinkName); err != nil { if oldLen := len(node.Labels); oldLen != 0 { delete(node.Labels, fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name)) delete(node.Labels, fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name)) diff --git a/pkg/daemon/init.go b/pkg/daemon/init.go index b82c41ec911..e096553adb3 100644 --- a/pkg/daemon/init.go +++ b/pkg/daemon/init.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/Wifx/gonetworkmanager" "github.com/vishvananda/netlink" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" @@ -15,6 +16,13 @@ import ( "github.com/kubeovn/kube-ovn/pkg/util" ) +var routeScopeOrders = [...]netlink.Scope{ + netlink.SCOPE_HOST, + netlink.SCOPE_LINK, + netlink.SCOPE_SITE, + netlink.SCOPE_UNIVERSE, +} + // InitOVSBridges initializes OVS bridges func InitOVSBridges() (map[string]string, error) { bridges, err := ovs.Bridges() @@ -100,10 +108,123 @@ func InitMirror(config *Configuration) error { return configureEmptyMirror(config.MirrorNic, config.MTU) } -func ovsInitProviderNetwork(provider, nic string) (int, error) { +func nmSetManaged(device string, managed bool) error { + nm, err := gonetworkmanager.NewNetworkManager() + if err != nil { + klog.V(5).Infof("failed to connect to NetworkManager: %v", err) + return nil + } + + d, err := nm.GetDeviceByIpIface(device) + if err != nil { + klog.Errorf("failed to get device by IP iface %s: %v", device, err) + return err + } + current, err := d.GetPropertyManaged() + if err != nil { + klog.Errorf("failed to get device property managed: %v", err) + return err + } + if current == managed { + return nil + } + + if err = d.SetPropertyManaged(managed); err != nil { + klog.Errorf("failed to set device property managed to %v: %v", managed, err) + return err + } + + return nil +} + +func changeProvideNicName(current, target string) (bool, error) { + link, err := netlink.LinkByName(current) + if err != nil { + if _, ok := err.(netlink.LinkNotFoundError); ok { + klog.Infof("link %s not found, skip", current) + return false, nil + } + klog.Errorf("failed to get link %s: %v", current, err) + return false, err + } + if link.Type() == "openvswitch" { + klog.Infof("%s is an openvswitch interface, skip", current) + return true, nil + } + + // set link unmanaged by NetworkManager to avoid getting new IP by DHCP + if err = nmSetManaged(current, false); err != nil { + klog.Errorf("failed set device %s to unmanaged by NetworkManager: %v", current, err) + return false, err + } + + klog.Infof("change nic name from %s to %s", current, target) + addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + klog.Errorf("failed to list addresses of link %s: %v", current, err) + return false, err + } + routes, err := netlink.RouteList(link, netlink.FAMILY_ALL) + if err != nil { + klog.Errorf("failed to list routes of link %s: %v", current, err) + return false, err + } + + if err = netlink.LinkSetDown(link); err != nil { + klog.Errorf("failed to set link %s down: %v", current, err) + return false, err + } + if err = netlink.LinkSetName(link, target); err != nil { + klog.Errorf("failed to set name of link %s to %s: %v", current, target, err) + return false, err + } + if err = netlink.LinkSetUp(link); err != nil { + klog.Errorf("failed to set link %s up: %v", target, err) + return false, err + } + + for _, addr := range addresses { + if addr.IP.IsLinkLocalUnicast() { + continue + } + if err = netlink.AddrReplace(link, &addr); err != nil { + klog.Errorf("failed to replace address %s: %v", addr.String(), err) + return false, err + } + } + + for _, scope := range routeScopeOrders { + for _, route := range routes { + if route.Gw == nil && route.Dst != nil && route.Dst.IP.IsLinkLocalUnicast() { + continue + } + if route.Scope == scope { + if err = netlink.RouteReplace(&route); err != nil { + klog.Errorf("failed to replace route %s: %v", route.String(), err) + return false, err + } + } + } + } + + return true, nil +} + +func ovsInitProviderNetwork(provider, nic string, exchangeLinkName bool) (int, error) { // create and configure external bridge brName := util.ExternalBridgeName(provider) - if err := configExternalBridge(provider, brName, nic); err != nil { + if exchangeLinkName { + exchanged, err := changeProvideNicName(nic, brName) + if err != nil { + klog.Errorf("failed to change provider nic name from %s to %s: %v", nic, brName, err) + return 0, err + } + if exchanged { + nic, brName = brName, nic + } + } + + if err := configExternalBridge(provider, brName, nic, exchangeLinkName); err != nil { errMsg := fmt.Errorf("failed to create and configure external bridge %s: %v", brName, err) klog.Error(errMsg) return 0, errMsg @@ -128,35 +249,52 @@ func ovsInitProviderNetwork(provider, nic string) (int, error) { } func ovsCleanProviderNetwork(provider string) error { - output, err := ovs.Exec("list-br") + output, err := ovs.Exec(ovs.IfExists, "get", "open", ".", "external-ids:ovn-bridge-mappings") if err != nil { + return fmt.Errorf("failed to get ovn-bridge-mappings, %v: %q", err, output) + } + + var idx int + var m, brName string + mappingPrefix := provider + ":" + brMappings := strings.Split(output, ",") + for idx, m = range brMappings { + if strings.HasPrefix(m, mappingPrefix) { + brName = m[len(mappingPrefix):] + break + } + } + + if output, err = ovs.Exec("list-br"); err != nil { return fmt.Errorf("failed to list OVS bridge %v: %q", err, output) } - brName := util.ExternalBridgeName(provider) if !util.ContainsString(strings.Split(output, "\n"), brName) { return nil } - if output, err = ovs.Exec(ovs.IfExists, "get", "open", ".", "external-ids:ovn-bridge-mappings"); err != nil { - return fmt.Errorf("failed to get ovn-bridge-mappings, %v: %q", err, output) + // get host nic + if output, err = ovs.Exec("list-ports", brName); err != nil { + return fmt.Errorf("failed to list ports of OVS bridge %s, %v: %q", brName, err, output) } - mappings := strings.Split(output, ",") - brMap := fmt.Sprintf("%s:%s", provider, brName) - - var idx int - for idx = range mappings { - if mappings[idx] == brMap { - break + // remove host nic from the external bridge + if output != "" { + for _, port := range strings.Split(output, "\n") { + if err = removeProviderNic(port, brName); err != nil { + errMsg := fmt.Errorf("failed to remove port %s from external bridge %s: %v", port, brName, err) + klog.Error(errMsg) + return errMsg + } } } - if idx != len(mappings) { - mappings = append(mappings[:idx], mappings[idx+1:]...) - if len(mappings) == 0 { + + if idx != len(brMappings) { + brMappings = append(brMappings[:idx], brMappings[idx+1:]...) + if len(brMappings) == 0 { output, err = ovs.Exec(ovs.IfExists, "remove", "open", ".", "external-ids", "ovn-bridge-mappings") } else { - output, err = ovs.Exec("set", "open", ".", "external-ids:ovn-bridge-mappings="+strings.Join(mappings, ",")) + output, err = ovs.Exec("set", "open", ".", "external-ids:ovn-bridge-mappings="+strings.Join(brMappings, ",")) } if err != nil { return fmt.Errorf("failed to set ovn-bridge-mappings, %v: %q", err, output) @@ -168,7 +306,7 @@ func ovsCleanProviderNetwork(provider string) error { } macMappings := strings.Split(output, ",") for _, macMap := range macMappings { - if len(macMap) == len(provider)+18 && strings.HasPrefix(macMap, provider) { + if strings.HasPrefix(macMap, mappingPrefix) { macMappings = util.RemoveString(macMappings, macMap) break } @@ -189,9 +327,9 @@ func ovsCleanProviderNetwork(provider string) error { // remove host nic from the external bridge if output != "" { - for _, nic := range strings.Split(output, "\n") { - if err = removeProviderNic(nic, brName); err != nil { - errMsg := fmt.Errorf("failed to remove nic %s from external bridge %s: %v", nic, brName, err) + for _, port := range strings.Split(output, "\n") { + if err = removeProviderNic(port, brName); err != nil { + errMsg := fmt.Errorf("failed to remove port %s from external bridge %s: %v", port, brName, err) klog.Error(errMsg) return errMsg } @@ -203,5 +341,12 @@ func ovsCleanProviderNetwork(provider string) error { return fmt.Errorf("failed to remove OVS bridge %s, %v: %q", brName, err, output) } + if br := util.ExternalBridgeName(provider); br != brName { + if _, err = changeProvideNicName(br, brName); err != nil { + klog.Errorf("failed to change provider nic name from %s to %s: %v", br, brName, err) + return err + } + } + return nil } diff --git a/pkg/daemon/ovs.go b/pkg/daemon/ovs.go index 18f4eaa8006..166686120e9 100644 --- a/pkg/daemon/ovs.go +++ b/pkg/daemon/ovs.go @@ -510,13 +510,15 @@ func configureNic(link, ip string, macAddr net.HardwareAddr, mtu int) error { return nil } -func configExternalBridge(provider, bridge, nic string) error { +func configExternalBridge(provider, bridge, nic string, exchangeLinkName bool) error { brExists, err := ovs.BridgeExists(bridge) if err != nil { return fmt.Errorf("failed to check OVS bridge existence: %v", err) } output, err := ovs.Exec(ovs.MayExist, "add-br", bridge, - "--", "set", "bridge", bridge, "external_ids:vendor="+util.CniTypeName) + "--", "set", "bridge", bridge, "external_ids:vendor="+util.CniTypeName, + "--", "set", "bridge", bridge, fmt.Sprintf("external_ids:exchange-link-name=%v", exchangeLinkName), + ) if err != nil { return fmt.Errorf("failed to create OVS bridge %s, %v: %q", bridge, err, output) } @@ -666,13 +668,7 @@ func configProviderNic(nicName, brName string) (int, error) { return 0, fmt.Errorf("failed to set OVS bridge %s up: %v", brName, err) } - scopeOrders := [...]netlink.Scope{ - netlink.SCOPE_HOST, - netlink.SCOPE_LINK, - netlink.SCOPE_SITE, - netlink.SCOPE_UNIVERSE, - } - for _, scope := range scopeOrders { + for _, scope := range routeScopeOrders { for _, route := range routes { if route.Gw == nil && route.Dst != nil && route.Dst.IP.IsLinkLocalUnicast() { // skip 169.254.0.0/16 and fe80::/10 diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 360af1c5eef..9bc50d7f34f 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -211,7 +211,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { providerInterface := underlay.UnderlayInterface if underlay.VlanID != "" { if vlanID, err = strconv.Atoi(underlay.VlanID); err != nil || vlanID <= 0 || vlanID > 4095 { - Fail(underlay.VlanID + " is not an invalid VLAN id") + Fail(underlay.VlanID + " is not a valid VLAN ID") } providerInterface = underlay.VlanInterface } @@ -289,6 +289,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { }, Spec: kubeovn.ProviderNetworkSpec{ DefaultInterface: providerInterface, + ExchangeLinkName: underlay.ExchangeLinkName, }, } if _, err = f.OvnClientSet.KubeovnV1().ProviderNetworks().Create(context.Background(), pn, metav1.CreateOptions{}); err != nil { diff --git a/test/e2e/underlay/underlay.go b/test/e2e/underlay/underlay.go index bd6d20739d8..fc8d1bd3772 100644 --- a/test/e2e/underlay/underlay.go +++ b/test/e2e/underlay/underlay.go @@ -3,6 +3,7 @@ package underlay import ( "context" "fmt" + "math/rand" "os" "os/exec" "strings" @@ -34,6 +35,8 @@ const ( ) var ( + ExchangeLinkName bool + VlanID = os.Getenv("VLAN_ID") cidr string @@ -45,6 +48,11 @@ var ( nodeMTU = make(map[string]int) ) +func init() { + rand.Seed(time.Now().UnixNano()) + ExchangeLinkName = rand.Intn(2) != 0 +} + func SetCIDR(s string) { cidr = s } @@ -100,7 +108,11 @@ var _ = Describe("[Underlay]", func() { } Expect(ovsPod).NotTo(BeNil()) - stdout, _, err := f.ExecToPodThroughAPI("ip addr show "+providerInterface, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil) + nic, br := providerInterface, util.ExternalBridgeName(ProviderNetwork) + if ExchangeLinkName { + nic, br = br, nic + } + stdout, _, err := f.ExecToPodThroughAPI("ip addr show "+nic, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil) Expect(err).NotTo(HaveOccurred()) addrNotFound := make([]bool, len(nodeAddrs[node.Name])) @@ -117,19 +129,19 @@ var _ = Describe("[Underlay]", func() { Expect(found).To(BeTrue()) } - stdout, _, err = f.ExecToPodThroughAPI("ovs-vsctl list-ports "+util.ExternalBridgeName(ProviderNetwork), "openvswitch", ovsPod.Name, ovsPod.Namespace, nil) + stdout, _, err = f.ExecToPodThroughAPI("ovs-vsctl list-ports "+br, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil) Expect(err).NotTo(HaveOccurred()) var portFound bool for _, port := range strings.Split(stdout, "\n") { - if port == providerInterface { + if port == nic { portFound = true break } } Expect(portFound).To(BeTrue()) - stdout, _, err = f.ExecToPodThroughAPI("ip addr show "+util.ExternalBridgeName(ProviderNetwork), "openvswitch", ovsPod.Name, ovsPod.Namespace, nil) + stdout, _, err = f.ExecToPodThroughAPI("ip addr show "+br, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil) Expect(err).NotTo(HaveOccurred()) var isUp bool diff --git a/yamls/crd.yaml b/yamls/crd.yaml index 4bae34bde22..3c079677aeb 100644 --- a/yamls/crd.yaml +++ b/yamls/crd.yaml @@ -327,6 +327,8 @@ spec: type: array items: type: string + exchangeLinkName: + type: boolean excludeNodes: type: array items: