Skip to content

Commit

Permalink
underlay: sync NetworkManager IP config to OVS bridge (#2949)
Browse files Browse the repository at this point in the history
  • Loading branch information
zhangzujian committed Jun 19, 2023
1 parent d9bab2e commit 738c407
Show file tree
Hide file tree
Showing 10 changed files with 317 additions and 80 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ require (
github.com/ovn-org/libovsdb v0.0.0-20221101143603-8f21d188c3a5
github.com/parnurzeal/gorequest v0.2.16
github.com/prometheus/client_golang v1.14.0
github.com/scylladb/go-set v1.0.2
github.com/sirupsen/logrus v1.9.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.1
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,8 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwC
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc=
github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
github.com/fatih/structtag v1.1.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
Expand Down Expand Up @@ -1226,6 +1228,8 @@ github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHi
github.com/satori/go.uuid v0.0.0-20160603004225-b111a074d5ef/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
github.com/scylladb/go-set v1.0.2 h1:SkvlMCKhP0wyyct6j+0IHJkBkSZL+TDzZ4E7f7BCcRE=
github.com/scylladb/go-set v1.0.2/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds=
Expand Down
6 changes: 3 additions & 3 deletions pkg/daemon/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ func (c *Controller) initProviderNetwork(pn *kubeovnv1.ProviderNetwork, node *v1

var mtu int
var err error
if mtu, err = ovsInitProviderNetwork(pn.Name, nic, pn.Spec.ExchangeLinkName, c.config.MacLearningFallback); err != nil {
if mtu, err = c.ovsInitProviderNetwork(pn.Name, nic, pn.Spec.ExchangeLinkName, c.config.MacLearningFallback); err != nil {
if oldLen := len(node.Labels); oldLen != 0 {
delete(node.Labels, fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name))
delete(node.Labels, fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name))
Expand Down Expand Up @@ -368,15 +368,15 @@ func (c *Controller) cleanProviderNetwork(pn *kubeovnv1.ProviderNetwork, node *v
return err
}

if err = ovsCleanProviderNetwork(pn.Name); err != nil {
if err = c.ovsCleanProviderNetwork(pn.Name); err != nil {
return err
}

return nil
}

func (c *Controller) handleDeleteProviderNetwork(pn *kubeovnv1.ProviderNetwork) error {
if err := ovsCleanProviderNetwork(pn.Name); err != nil {
if err := c.ovsCleanProviderNetwork(pn.Name); err != nil {
return err
}

Expand Down
5 changes: 5 additions & 0 deletions pkg/daemon/controller_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ type ControllerRuntime struct {
iptables map[string]*iptables.IPTables
iptablesObsolete map[string]*iptables.IPTables
ipsets map[string]*ipsets.IPSets

nmSyncer *networkManagerSyncer
}

func evalCommandSymlinks(cmd string) (string, error) {
Expand Down Expand Up @@ -103,6 +105,9 @@ func (c *Controller) initRuntime() error {
c.ipsets[kubeovnv1.ProtocolIPv6] = ipsets.NewIPSets(ipsets.NewIPVersionConfig(ipsets.IPFamilyV6, IPSetPrefix, nil, nil))
}

c.nmSyncer = newNetworkManagerSyncer()
c.nmSyncer.Run(c.transferAddrsAndRoutes)

return nil
}

Expand Down
14 changes: 5 additions & 9 deletions pkg/daemon/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,6 @@ func InitOVSBridges() (map[string]string, error) {
return nil, fmt.Errorf("failed to check vendor of port %s: %v", port, err)
}
if ok {
klog.Infof("config provider nic %s on bridge %s", port, brName)
if _, err = configProviderNic(port, brName); err != nil {
return nil, err
}
mappings[port] = brName
}
}
Expand Down Expand Up @@ -97,11 +93,11 @@ func InitMirror(config *Configuration) error {
return configureEmptyMirror(config.MirrorNic, config.MTU)
}

func ovsInitProviderNetwork(provider, nic string, exchangeLinkName, macLearningFallback bool) (int, error) {
func (c *Controller) ovsInitProviderNetwork(provider, nic string, exchangeLinkName, macLearningFallback bool) (int, error) {
// create and configure external bridge
brName := util.ExternalBridgeName(provider)
if exchangeLinkName {
exchanged, err := changeProvideNicName(nic, brName)
exchanged, err := c.changeProvideNicName(nic, brName)
if err != nil {
klog.Errorf("failed to change provider nic name from %s to %s: %v", nic, brName, err)
return 0, err
Expand All @@ -126,7 +122,7 @@ func ovsInitProviderNetwork(provider, nic string, exchangeLinkName, macLearningF

// add host nic to the external bridge
klog.Infof("config provider nic %s on bridge %s", nic, brName)
mtu, err := configProviderNic(nic, brName)
mtu, err := c.configProviderNic(nic, brName)
if err != nil {
errMsg := fmt.Errorf("failed to add nic %s to external bridge %s: %v", nic, brName, err)
klog.Error(errMsg)
Expand All @@ -136,7 +132,7 @@ func ovsInitProviderNetwork(provider, nic string, exchangeLinkName, macLearningF
return mtu, nil
}

func ovsCleanProviderNetwork(provider string) error {
func (c *Controller) ovsCleanProviderNetwork(provider string) error {
mappings, err := getOvnMappings("ovn-bridge-mappings")
if err != nil {
return err
Expand Down Expand Up @@ -189,7 +185,7 @@ func ovsCleanProviderNetwork(provider string) error {
klog.V(3).Infof("ovs bridge %s has been deleted", brName)

if br := util.ExternalBridgeName(provider); br != brName {
if _, err = changeProvideNicName(br, brName); err != nil {
if _, err = c.changeProvideNicName(br, brName); err != nil {
klog.Errorf("failed to change provider nic name from %s to %s: %v", br, brName, err)
return err
}
Expand Down
47 changes: 3 additions & 44 deletions pkg/daemon/init_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package daemon
import (
"time"

"github.com/kubeovn/gonetworkmanager/v2"
"github.com/vishvananda/netlink"
"k8s.io/klog/v2"

Expand All @@ -17,46 +16,6 @@ var routeScopeOrders = [...]netlink.Scope{
netlink.SCOPE_UNIVERSE,
}

func nmSetManaged(device string, managed bool) error {
nm, err := gonetworkmanager.NewNetworkManager()
if err != nil {
klog.V(5).Infof("failed to connect to NetworkManager: %v", err)
return nil
}

running, err := nm.Running()
if err != nil {
klog.Warningf("failed to check NetworkManager running state: %v", err)
return nil
}
if !running {
klog.V(5).Info("NetworkManager is not running, ignore")
return nil
}

d, err := nm.GetDeviceByIpIface(device)
if err != nil {
klog.Errorf("failed to get device by IP iface %s: %v", device, err)
return err
}
current, err := d.GetPropertyManaged()
if err != nil {
klog.Errorf("failed to get device property managed: %v", err)
return err
}
if current == managed {
return nil
}

klog.Infof(`setting device %s NetworkManager property "managed" to %v`, device, managed)
if err = d.SetPropertyManaged(managed); err != nil {
klog.Errorf("failed to set device property managed to %v: %v", managed, err)
return err
}

return nil
}

// wait systemd-networkd to finish interface configuration
func waitNetworkdConfiguration(linkIndex int) {
done := make(chan struct{})
Expand Down Expand Up @@ -90,7 +49,7 @@ func waitNetworkdConfiguration(linkIndex int) {
}
}

func changeProvideNicName(current, target string) (bool, error) {
func (c *Controller) changeProvideNicName(current, target string) (bool, error) {
link, err := netlink.LinkByName(current)
if err != nil {
if _, ok := err.(netlink.LinkNotFoundError); ok {
Expand All @@ -117,8 +76,8 @@ func changeProvideNicName(current, target string) (bool, error) {
}

// set link unmanaged by NetworkManager
if err = nmSetManaged(current, false); err != nil {
klog.Errorf("failed set device %s unmanaged by NetworkManager: %v", current, err)
if err = c.nmSyncer.SetManaged(current, false); err != nil {
klog.Errorf("failed to set device %s unmanaged by NetworkManager: %v", current, err)
return false, err
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/daemon/init_windows.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package daemon

func changeProvideNicName(nic, br string) (bool, error) {
func (c *Controller) changeProvideNicName(nic, br string) (bool, error) {
// not supported on windows
return false, nil
}
186 changes: 186 additions & 0 deletions pkg/daemon/nm_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
package daemon

import (
"sync"

"github.com/kubeovn/gonetworkmanager/v2"
"github.com/scylladb/go-set/strset"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog/v2"
)

type networkManagerSyncer struct {
manager gonetworkmanager.NetworkManager
workqueue workqueue.Interface
devicePaths *strset.Set
pathMap map[string]string
bridgeMap map[string]string
lock sync.Mutex
}

func newNetworkManagerSyncer() *networkManagerSyncer {
syncer := &networkManagerSyncer{}

manager, err := gonetworkmanager.NewNetworkManager()
if err != nil {
klog.V(3).Infof("failed to connect to NetworkManager: %v", err)
return syncer
}

running, err := manager.Running()
if err != nil {
klog.V(3).Infof("failed to check NetworkManager running state: %v", err)
return syncer
}
if !running {
klog.V(3).Info("NetworkManager is not running, ignore")
return syncer
}

syncer.manager = manager
syncer.workqueue = workqueue.NewNamed("NetworkManagerSyncer")
syncer.devicePaths = strset.New()
syncer.pathMap = make(map[string]string)
syncer.bridgeMap = make(map[string]string)
return syncer
}

func (n *networkManagerSyncer) Run(handler func(nic, bridge string, delNonExistent bool) (int, error)) {
if n.manager == nil {
return
}

go func() {
for n.ProcessNextItem(handler) {
}
}()

go func() {
ch := n.manager.Subscribe()
defer n.manager.Unsubscribe()

stateChange := gonetworkmanager.DeviceInterface + "." + gonetworkmanager.ActiveConnectionSignalStateChanged
for {
event := <-ch

n.lock.Lock()
if len(event.Body) == 0 || event.Name != stateChange || !n.devicePaths.Has(string(event.Path)) {
n.lock.Unlock()
continue
}
n.lock.Unlock()

state, ok := event.Body[0].(uint32)
if !ok {
klog.Warningf("failed to convert %#v to uint32", event.Body[0])
continue
}
if gonetworkmanager.NmDeviceState(state) != gonetworkmanager.NmDeviceStateActivated {
continue
}

klog.Infof("adding dbus object path %s to workqueue", event.Path)
n.workqueue.Add(string(event.Path))
}
}()
}

func (n *networkManagerSyncer) ProcessNextItem(handler func(nic, bridge string, delNonExistent bool) (int, error)) bool {
item, shutdown := n.workqueue.Get()
if shutdown {
return false
}
defer n.workqueue.Done(item)

klog.Infof("process dbus object path %v", item)
path := item.(string)
n.lock.Lock()
if !n.devicePaths.Has(path) {
n.lock.Unlock()
return true
}
var nic string
for k, v := range n.pathMap {
if v == path {
nic = k
break
}
}
n.lock.Unlock()

bridge := n.bridgeMap[nic]
if _, err := handler(nic, bridge, true); err != nil {
klog.Errorf("failed to handle NetworkManager event for device %s with bridge %s: %v", nic, bridge, err)
}

return true
}

func (n *networkManagerSyncer) AddDevice(nicName, bridge string) error {
if n.manager == nil {
return nil
}

n.lock.Lock()
defer n.lock.Unlock()

if _, ok := n.pathMap[nicName]; ok {
return nil
}

device, err := n.manager.GetDeviceByIpIface(nicName)
if err != nil {
klog.Errorf("failed to get device by IP iface %q: %v", nicName, err)
return err
}

path := string(device.GetPath())
klog.V(3).Infof("adding device %s with dbus object path %s and bridge %s", nicName, path, bridge)
n.devicePaths.Add(path)
n.pathMap[nicName] = path
n.bridgeMap[nicName] = bridge

return nil
}

func (n *networkManagerSyncer) RemoveDevice(nicName string) error {
if n.manager == nil {
return nil
}

n.lock.Lock()
n.devicePaths.Remove(n.pathMap[nicName])
delete(n.pathMap, nicName)
delete(n.bridgeMap, nicName)
n.lock.Unlock()

return nil
}

func (n *networkManagerSyncer) SetManaged(name string, managed bool) error {
if n.manager == nil {
return nil
}

device, err := n.manager.GetDeviceByIpIface(name)
if err != nil {
klog.Errorf("failed to get device by IP iface %q: %v", name, err)
return err
}
current, err := device.GetPropertyManaged()
if err != nil {
klog.Errorf("failed to get device property managed: %v", err)
return err
}
if current == managed {
return nil
}

klog.Infof(`setting device %s NetworkManager property "managed" to %v`, name, managed)
if err = device.SetPropertyManaged(managed); err != nil {
klog.Errorf("failed to set device property managed to %v: %v", managed, err)
return err
}

return nil
}
Loading

0 comments on commit 738c407

Please sign in to comment.