Skip to content

Commit

Permalink
Refactor InterfaceStore
Browse files Browse the repository at this point in the history 
Move InterfaceStore to a separate package. This could avoid
sub-packages of agent depends on the agent package which could lead
to import cycle.
For the same reason, move NodeConfig and GatewayConfig from agent to
agent/types.
Move OVS port data initialization to agent.Initializer.
Move OVS port external ID generation and parsing to podConfigurator.
Change "MTU" in structs and func arguments to "mtu".
  • Loading branch information
@jianjuns @antoninbas
jianjuns authored and antoninbas committed Nov 28, 2019
1 parent b963dc7 commit a71140c
Show file tree
Hide file tree
Showing 16 changed files with 499 additions and 417 deletions.
3 changes: 2 additions & 1 deletion cmd/antrea-agent/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
_ "github.com/vmware-tanzu/antrea/pkg/agent/cniserver/ipam"
"github.com/vmware-tanzu/antrea/pkg/agent/controller/networkpolicy"
"github.com/vmware-tanzu/antrea/pkg/agent/controller/noderoute"
"github.com/vmware-tanzu/antrea/pkg/agent/interfacestore"
"github.com/vmware-tanzu/antrea/pkg/agent/openflow"
"github.com/vmware-tanzu/antrea/pkg/k8s"
"github.com/vmware-tanzu/antrea/pkg/monitor"
Expand Down Expand Up @@ -67,7 +68,7 @@ func run(o *Options) error {
ofClient := openflow.NewClient(o.config.OVSBridge)

// Create an ifaceStore that caches network interfaces managed by this node.
ifaceStore := agent.NewInterfaceStore()
ifaceStore := interfacestore.NewInterfaceStore()

// Initialize agent and node network.
agentInitializer := agent.NewInitializer(
Expand Down
89 changes: 60 additions & 29 deletions pkg/agent/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,11 @@ import (
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog"

"github.com/vmware-tanzu/antrea/pkg/agent/cniserver"
"github.com/vmware-tanzu/antrea/pkg/agent/interfacestore"
"github.com/vmware-tanzu/antrea/pkg/agent/iptables"
"github.com/vmware-tanzu/antrea/pkg/agent/openflow"
"github.com/vmware-tanzu/antrea/pkg/agent/types"
"github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig"
)

Expand All @@ -41,29 +44,16 @@ const (
IPSecPSKEnvKey = "ANTREA_IPSEC_PSK"
)

type NodeConfig struct {
Bridge string
Name string
PodCIDR *net.IPNet
*Gateway
}

type Gateway struct {
IP net.IP
MAC net.HardwareAddr
Name string
}

// Initializer knows how to setup host networking, OpenVSwitch, and Openflow.
type Initializer struct {
ovsBridge string
hostGateway string
tunnelType string
MTU int
mtu int
enableIPSecTunnel bool
client clientset.Interface
ifaceStore InterfaceStore
nodeConfig *NodeConfig
ifaceStore interfacestore.InterfaceStore
nodeConfig *types.NodeConfig
ovsBridgeClient ovsconfig.OVSBridgeClient
serviceCIDR *net.IPNet
ofClient openflow.Client
Expand All @@ -84,9 +74,9 @@ func NewInitializer(
ovsBridgeClient ovsconfig.OVSBridgeClient,
ofClient openflow.Client,
k8sClient clientset.Interface,
ifaceStore InterfaceStore,
ifaceStore interfacestore.InterfaceStore,
ovsBridge, serviceCIDR, hostGateway, tunnelType string,
MTU int,
mtu int,
enableIPSecTunnel bool) *Initializer {
// Parse service CIDR configuration. serviceCIDR is checked in option.validate, so
// it should be a valid configuration here.
Expand All @@ -96,7 +86,7 @@ func NewInitializer(
ovsBridge: ovsBridge,
hostGateway: hostGateway,
tunnelType: tunnelType,
MTU: MTU,
mtu: mtu,
enableIPSecTunnel: enableIPSecTunnel,
client: k8sClient,
ifaceStore: ifaceStore,
Expand All @@ -106,7 +96,7 @@ func NewInitializer(
}

// GetNodeConfig returns the NodeConfig.
func (i *Initializer) GetNodeConfig() *NodeConfig {
func (i *Initializer) GetNodeConfig() *types.NodeConfig {
return i.nodeConfig
}

Expand All @@ -123,7 +113,7 @@ func (i *Initializer) setupOVSBridge() error {
}

// Initialize interface cache
if err := i.ifaceStore.Initialize(i.ovsBridgeClient, i.hostGateway, TunPortName); err != nil {
if err := i.initInterfaceStore(); err != nil {
return err
}

Expand All @@ -150,6 +140,47 @@ func (i *Initializer) setupOVSBridge() error {
return nil
}

// initInterfaceStore initializes InterfaceStore with all OVS ports retrieved
// from the OVS bridge.
func (i *Initializer) initInterfaceStore() error {
ovsPorts, err := i.ovsBridgeClient.GetPortList()
if err != nil {
klog.Errorf("Failed to list OVS ports: %v", err)
return err
}

ifaceList := make([]*interfacestore.InterfaceConfig, 0, len(ovsPorts))
for index := range ovsPorts {
port := &ovsPorts[index]
ovsPort := &interfacestore.OVSPortConfig{
IfaceName: port.Name,
PortUUID: port.UUID,
OFPort: port.OFPort}
var intf *interfacestore.InterfaceConfig
switch {
case port.Name == i.hostGateway:
intf = &interfacestore.InterfaceConfig{
Type: interfacestore.GatewayInterface,
OVSPortConfig: ovsPort,
ID: i.hostGateway}
case port.Name == TunPortName:
intf = &interfacestore.InterfaceConfig{
Type: interfacestore.TunnelInterface,
OVSPortConfig: ovsPort,
ID: TunPortName}
default:
// The port should be for a container interface.
intf = cniserver.ParseOVSPortInterfaceConfig(port, ovsPort)
}
if intf != nil {
ifaceList = append(ifaceList, intf)
}
}

i.ifaceStore.Initialize(ifaceList)
return nil
}

func (i *Initializer) Initialize() error {
klog.Info("Setting up node network")
if err := i.initNodeLocalConfig(); err != nil {
Expand Down Expand Up @@ -229,17 +260,17 @@ func (i *Initializer) setupGatewayInterface() error {
klog.Errorf("Failed to add host interface %s on OVS: %v", i.hostGateway, err)
return err
}
gatewayIface = NewGatewayInterface(i.hostGateway)
gatewayIface.OVSPortConfig = &OVSPortConfig{i.hostGateway, gwPortUUID, hostGatewayOFPort}
gatewayIface = interfacestore.NewGatewayInterface(i.hostGateway)
gatewayIface.OVSPortConfig = &interfacestore.OVSPortConfig{i.hostGateway, gwPortUUID, hostGatewayOFPort}
i.ifaceStore.AddInterface(i.hostGateway, gatewayIface)
} else {
klog.V(2).Infof("Gateway port %s already exists on OVS bridge", i.hostGateway)
}
// Idempotent operation to set the gateway's MTU: we perform this operation regardless of
// whether or not the gateway interface already exists, as the desired MTU may change across
// restarts.
klog.V(4).Infof("Setting gateway interface %s MTU to %d", i.hostGateway, i.MTU)
i.ovsBridgeClient.SetInterfaceMTU(i.hostGateway, i.MTU)
klog.V(4).Infof("Setting gateway interface %s MTU to %d", i.hostGateway, i.mtu)
i.ovsBridgeClient.SetInterfaceMTU(i.hostGateway, i.mtu)
// host link might not be queried at once after create OVS internal port, retry max 5 times with 1s
// delay each time to ensure the link is ready. If still failed after max retry return error.
link, err := func() (netlink.Link, error) {
Expand Down Expand Up @@ -274,7 +305,7 @@ func (i *Initializer) setupGatewayInterface() error {
gwIP := &net.IPNet{IP: ip.NextIP(subnetID), Mask: localSubnet.Mask}
gwAddr := &netlink.Addr{IPNet: gwIP, Label: ""}
gwMAC := link.Attrs().HardwareAddr
i.nodeConfig.Gateway = &Gateway{Name: i.hostGateway, IP: gwIP.IP, MAC: gwMAC}
i.nodeConfig.GatewayConfig = &types.GatewayConfig{Name: i.hostGateway, IP: gwIP.IP, MAC: gwMAC}
gatewayIface.IP = gwIP.IP
gatewayIface.MAC = gwMAC

Expand Down Expand Up @@ -326,8 +357,8 @@ func (i *Initializer) setupTunnelInterface(tunnelPortName string) error {
klog.Errorf("Failed to add tunnel port %s type %s on OVS: %v", tunnelPortName, i.tunnelType, err)
return err
}
tunnelIface = NewTunnelInterface(tunnelPortName)
tunnelIface.OVSPortConfig = &OVSPortConfig{tunnelPortName, tunnelPortUUID, tunOFPort}
tunnelIface = interfacestore.NewTunnelInterface(tunnelPortName)
tunnelIface.OVSPortConfig = &interfacestore.OVSPortConfig{tunnelPortName, tunnelPortUUID, tunOFPort}
i.ifaceStore.AddInterface(tunnelPortName, tunnelIface)
return nil
}
Expand Down Expand Up @@ -356,7 +387,7 @@ func (i *Initializer) initNodeLocalConfig() error {
return err
}

i.nodeConfig = &NodeConfig{Name: nodeName, PodCIDR: localSubnet}
i.nodeConfig = &types.NodeConfig{Name: nodeName, PodCIDR: localSubnet}
return nil
}

Expand Down
79 changes: 79 additions & 0 deletions pkg/agent/agent_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,18 @@
package agent

import (
"fmt"
"net"
"os"
"testing"

mock "github.com/golang/mock/gomock"
"github.com/google/uuid"

"github.com/vmware-tanzu/antrea/pkg/agent/cniserver"
"github.com/vmware-tanzu/antrea/pkg/agent/interfacestore"
"github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig"
ovsconfigtest "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig/testing"
)

func TestGetNodeName(t *testing.T) {
Expand Down Expand Up @@ -50,3 +60,72 @@ func compareNodeName(k, v string, t *testing.T) {
t.Errorf("Failed to retrieve nodename, want: %s, get: %s", v, nodeName)
}
}

func newAgentInitializer(ovsBridgeClient ovsconfig.OVSBridgeClient, ifaceStore interfacestore.InterfaceStore) *Initializer {
return &Initializer{ovsBridgeClient: ovsBridgeClient, ifaceStore: ifaceStore, hostGateway: "gw0"}
}

func convertExternalIDMap(in map[string]interface{}) map[string]string {
out := make(map[string]string, len(in))
for k, v := range in {
out[k] = v.(string)
}
return out
}

func TestInitstore(t *testing.T) {
controller := mock.NewController(t)
defer controller.Finish()
mockOVSBridgeClient := ovsconfigtest.NewMockOVSBridgeClient(controller)

mockOVSBridgeClient.EXPECT().GetPortList().Return(nil, ovsconfig.NewTransactionError(fmt.Errorf("Failed to list OVS ports"), true))

store := interfacestore.NewInterfaceStore()
initializer := newAgentInitializer(mockOVSBridgeClient, store)

err := initializer.initInterfaceStore()
if err == nil {
t.Errorf("Failed to handle OVS return error")
}

uuid1 := uuid.New().String()
uuid2 := uuid.New().String()
p1MAC := "11:22:33:44:55:66"
p1IP := "1.1.1.1"
p2MAC := "11:22:33:44:55:77"
p2IP := "1.1.1.2"
p1NetMAC, _ := net.ParseMAC(p1MAC)
p1NetIP := net.ParseIP(p1IP)
p2NetMAC, _ := net.ParseMAC(p2MAC)
p2NetIP := net.ParseIP(p2IP)

ovsPort1 := ovsconfig.OVSPortData{UUID: uuid1, Name: "p1", IFName: "p1", OFPort: 1,
ExternalIDs: convertExternalIDMap(cniserver.BuildOVSPortExternalIDs(
interfacestore.NewContainerInterface(uuid1, "pod1", "ns1", "netns1", p1NetMAC, p1NetIP)))}
ovsPort2 := ovsconfig.OVSPortData{UUID: uuid2, Name: "p2", IFName: "p2", OFPort: 2,
ExternalIDs: convertExternalIDMap(cniserver.BuildOVSPortExternalIDs(
interfacestore.NewContainerInterface(uuid2, "pod2", "ns2", "netns2", p2NetMAC, p2NetIP)))}
initOVSPorts := []ovsconfig.OVSPortData{ovsPort1, ovsPort2}

mockOVSBridgeClient.EXPECT().GetPortList().Return(initOVSPorts, ovsconfig.NewTransactionError(fmt.Errorf("Failed to list OVS ports"), true))
err = initializer.initInterfaceStore()
if store.Len() != 0 {
t.Errorf("Failed to load OVS port in store")
}

mockOVSBridgeClient.EXPECT().GetPortList().Return(initOVSPorts, nil)
err = initializer.initInterfaceStore()
if store.Len() != 2 {
t.Errorf("Failed to load OVS port in store")
}
container1, found1 := store.GetInterface("p1")
if !found1 {
t.Errorf("Failed to load OVS port into local store")
} else if container1.OFPort != 1 || container1.IP.String() != p1IP || container1.MAC.String() != p1MAC || container1.IfaceName != "p1" {
t.Errorf("Failed to load OVS port configuration into local store")
}
_, found2 := store.GetInterface("p2")
if !found2 {
t.Errorf("Failed to load OVS port into local store")
}
}
Loading

0 comments on commit a71140c

Please sign in to comment.