diff --git a/aitelemetry/telemetrywrapper_test.go b/aitelemetry/telemetrywrapper_test.go index 4031f6d9fe..5fda52de70 100644 --- a/aitelemetry/telemetrywrapper_test.go +++ b/aitelemetry/telemetrywrapper_test.go @@ -29,12 +29,15 @@ func TestMain(m *testing.M) { fmt.Printf("TestST LogDir configuration succeeded\n") } + p := platform.NewExecClient() if runtime.GOOS == "linux" { - platform.ExecuteCommand("cp metadata_test.json /tmp/azuremetadata.json") + //nolint:errcheck // initial test setup + p.ExecuteCommand("cp metadata_test.json /tmp/azuremetadata.json") } else { metadataFile := filepath.FromSlash(os.Getenv("TEMP")) + "\\azuremetadata.json" cmd := fmt.Sprintf("copy metadata_test.json %s", metadataFile) - platform.ExecuteCommand(cmd) + //nolint:errcheck // initial test setup + p.ExecuteCommand(cmd) } hostu, _ := url.Parse("tcp://" + hostAgentUrl) @@ -54,11 +57,13 @@ func TestMain(m *testing.M) { exitCode := m.Run() if runtime.GOOS == "linux" { - platform.ExecuteCommand("rm /tmp/azuremetadata.json") + //nolint:errcheck // test cleanup + p.ExecuteCommand("rm /tmp/azuremetadata.json") } else { metadataFile := filepath.FromSlash(os.Getenv("TEMP")) + "\\azuremetadata.json" cmd := fmt.Sprintf("del %s", metadataFile) - platform.ExecuteCommand(cmd) + //nolint:errcheck // initial test cleanup + p.ExecuteCommand(cmd) } log.Close() diff --git a/cni/network/network.go b/cni/network/network.go index 971fc00cf5..92c9e0494f 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -114,7 +114,7 @@ func NewPlugin(name string, nl := netlink.NewNetlink() // Setup network manager. - nm, err := network.NewNetworkManager(nl) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient()) if err != nil { return nil, err } diff --git a/cnm/network/network.go b/cnm/network/network.go index dee59881fa..e811f4d098 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -52,7 +52,7 @@ func NewPlugin(config *common.PluginConfig) (NetPlugin, error) { nl := netlink.NewNetlink() // Setup network manager. - nm, err := network.NewNetworkManager(nl) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient()) if err != nil { return nil, err } diff --git a/cnms/service/networkmonitor.go b/cnms/service/networkmonitor.go index d01c84e437..6341bc06ee 100644 --- a/cnms/service/networkmonitor.go +++ b/cnms/service/networkmonitor.go @@ -148,7 +148,7 @@ func main() { } nl := netlink.NewNetlink() - nm, err := network.NewNetworkManager(nl) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient()) if err != nil { log.Printf("[monitor] Failed while creating network manager") return diff --git a/cns/dockerclient/dockerclient.go b/cns/dockerclient/dockerclient.go index 1c43cada6f..b28c108084 100644 --- a/cns/dockerclient/dockerclient.go +++ b/cns/dockerclient/dockerclient.go @@ -150,6 +150,7 @@ func (dockerClient *DockerClient) CreateNetwork(networkName string, nicInfo *imd // DeleteNetwork creates a network using docker network create. func (dockerClient *DockerClient) DeleteNetwork(networkName string) error { + p := platform.NewExecClient() logger.Printf("[Azure CNS] DeleteNetwork") url := dockerClient.connectionURL + inspectNetworkPath + networkName @@ -178,7 +179,7 @@ func (dockerClient *DockerClient) DeleteNetwork(networkName string) error { cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d %v -j MASQUERADE", primaryNic.Subnet) - _, err = platform.ExecuteCommand(cmd) + _, err = p.ExecuteCommand(cmd) if err != nil { logger.Printf("[Azure CNS] Error Removing Outbound SNAT rule %v", err) } diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go index 39d4c94337..ba8e215622 100644 --- a/ebtables/ebtables.go +++ b/ebtables/ebtables.go @@ -129,11 +129,11 @@ func GetEbtableRules(tableName, chainName string) ([]string, error) { inChain bool rules []string ) - + p := platform.NewExecClient() command := fmt.Sprintf( "ebtables -t %s -L %s --Lmac2", tableName, chainName) - out, err := platform.ExecuteCommand(command) + out, err := p.ExecuteCommand(command) if err != nil { return nil, err } @@ -226,8 +226,9 @@ func EbTableRuleExists(tableName, chainName, matchSet string) (bool, error) { // runEbCmd runs an EB rule command. func runEbCmd(table, action, chain, rule string) error { + p := platform.NewExecClient() command := fmt.Sprintf("ebtables -t %s %s %s %s", table, action, chain, rule) - _, err := platform.ExecuteCommand(command) + _, err := p.ExecuteCommand(command) return err } diff --git a/ipam/ipv6Ipam.go b/ipam/ipv6Ipam.go index 7db99b8e5c..a69350feb1 100644 --- a/ipam/ipv6Ipam.go +++ b/ipam/ipv6Ipam.go @@ -153,6 +153,7 @@ func (source *ipv6IpamSource) refresh() error { } if source.isLoaded { + log.Printf("ipv6 source already loaded") return nil } @@ -212,7 +213,9 @@ func (source *ipv6IpamSource) refresh() error { return err } + source.isLoaded = true log.Printf("[ipam] Address space successfully populated from Kubernetes API Server") + return err } diff --git a/iptables/iptables.go b/iptables/iptables.go index dc0a6e8c0c..79d1b28fdf 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -94,6 +94,7 @@ type IPTableEntry struct { func RunCmd(version, params string) error { var cmd string + p := platform.NewExecClient() iptCmd := iptables if version == V6 { iptCmd = ip6tables @@ -105,7 +106,7 @@ func RunCmd(version, params string) error { cmd = fmt.Sprintf("%s -w %d %s", iptCmd, lockTimeout, params) } - if _, err := platform.ExecuteCommand(cmd); err != nil { + if _, err := p.ExecuteCommand(cmd); err != nil { return err } diff --git a/netlink/ip.go b/netlink/ip_linux.go similarity index 100% rename from netlink/ip.go rename to netlink/ip_linux.go diff --git a/netlink/link.go b/netlink/link_linux.go similarity index 100% rename from netlink/link.go rename to netlink/link_linux.go diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go index a068f91fab..1150e37bfb 100644 --- a/network/bridge_endpointclient_linux.go +++ b/network/bridge_endpointclient_linux.go @@ -6,8 +6,10 @@ import ( "github.com/Azure/azure-container-networking/ebtables" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/platform" ) const ( @@ -26,6 +28,9 @@ type LinuxBridgeEndpointClient struct { hostIPAddresses []*net.IPNet mode string netlink netlink.NetlinkInterface + plClient platform.ExecClient + netioshim netio.NetIOInterface + nuc networkutils.NetworkUtils } func NewLinuxBridgeEndpointClient( @@ -34,6 +39,7 @@ func NewLinuxBridgeEndpointClient( containerVethName string, mode string, nl netlink.NetlinkInterface, + plc platform.ExecClient, ) *LinuxBridgeEndpointClient { client := &LinuxBridgeEndpointClient{ @@ -45,16 +51,17 @@ func NewLinuxBridgeEndpointClient( hostIPAddresses: []*net.IPNet{}, mode: mode, netlink: nl, + plClient: plc, + netioshim: &netio.NetIO{}, } client.hostIPAddresses = append(client.hostIPAddresses, extIf.IPAddresses...) - + client.nuc = networkutils.NewNetworkUtils(nl, plc) return client } func (client *LinuxBridgeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { + if err := client.nuc.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { return err } @@ -164,8 +171,7 @@ func (client *LinuxBridgeEndpointClient) MoveEndpointsToContainerNS(epInfo *Endp } func (client *LinuxBridgeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + if err := client.nuc.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return err } @@ -175,19 +181,11 @@ func (client *LinuxBridgeEndpointClient) SetupContainerInterfaces(epInfo *Endpoi } func (client *LinuxBridgeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - if epInfo.IPV6Mode != "" { - // Enable ipv6 setting in container - if err := epcommon.UpdateIPV6Setting(0); err != nil { - return err - } - } - - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + if err := client.nuc.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return err } - if err := addRoutes(client.netlink, client.containerVethName, epInfo.Routes); err != nil { + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, epInfo.Routes); err != nil { return err } @@ -280,7 +278,7 @@ func (client *LinuxBridgeEndpointClient) setupIPV6Routes(epInfo *EndpointInfo) e routes = append(routes, defaultV6Route) log.Printf("[net] Adding ipv6 routes in container %+v", routes) - if err := addRoutes(client.netlink, client.containerVethName, routes); err != nil { + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, routes); err != nil { return nil } } diff --git a/network/bridge_networkclient_linux.go b/network/bridge_networkclient_linux.go index 3ffadbf2e2..25893dc41c 100644 --- a/network/bridge_networkclient_linux.go +++ b/network/bridge_networkclient_linux.go @@ -8,7 +8,8 @@ import ( "github.com/Azure/azure-container-networking/ebtables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/platform" ) const ( @@ -26,6 +27,7 @@ type LinuxBridgeClient struct { hostInterfaceName string nwInfo NetworkInfo netlink netlink.NetlinkInterface + nuClient networkutils.NetworkUtils } func NewLinuxBridgeClient( @@ -33,12 +35,14 @@ func NewLinuxBridgeClient( hostInterfaceName string, nwInfo NetworkInfo, nl netlink.NetlinkInterface, + plc platform.ExecClient, ) *LinuxBridgeClient { client := &LinuxBridgeClient{ bridgeName: bridgeName, nwInfo: nwInfo, hostInterfaceName: hostInterfaceName, netlink: nl, + nuClient: networkutils.NewNetworkUtils(nl, plc), } return client @@ -58,7 +62,11 @@ func (client *LinuxBridgeClient) CreateBridge() error { return err } - return epcommon.DisableRAForInterface(client.bridgeName) + if err := client.nuClient.DisableRAForInterface(client.bridgeName); err != nil { + return fmt.Errorf("CreateBridge:%w", err) + } + + return nil } func (client *LinuxBridgeClient) DeleteBridge() error { @@ -123,7 +131,7 @@ func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error { return err } - if err := epcommon.EnableIPV6Forwarding(); err != nil { + if err := client.nuClient.EnableIPV6Forwarding(); err != nil { return err } } diff --git a/network/endpoint.go b/network/endpoint.go index 740f80e611..f54363e1bb 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/policy" + "github.com/Azure/azure-container-networking/platform" ) const ( @@ -98,7 +99,7 @@ type apipaClient interface { } // NewEndpoint creates a new endpoint in the network. -func (nw *network) newEndpoint(cli apipaClient, nl netlink.NetlinkInterface, epInfo *EndpointInfo) (*endpoint, error) { +func (nw *network) newEndpoint(cli apipaClient, nl netlink.NetlinkInterface, plc platform.ExecClient, epInfo *EndpointInfo) (*endpoint, error) { var ep *endpoint var err error @@ -110,7 +111,7 @@ func (nw *network) newEndpoint(cli apipaClient, nl netlink.NetlinkInterface, epI }() // Call the platform implementation. - ep, err = nw.newEndpointImpl(cli, nl, epInfo) + ep, err = nw.newEndpointImpl(cli, nl, plc, epInfo) if err != nil { return nil, err } @@ -122,7 +123,7 @@ func (nw *network) newEndpoint(cli apipaClient, nl netlink.NetlinkInterface, epI } // DeleteEndpoint deletes an existing endpoint from the network. -func (nw *network) deleteEndpoint(cli apipaClient, nl netlink.NetlinkInterface, endpointID string) error { +func (nw *network) deleteEndpoint(cli apipaClient, nl netlink.NetlinkInterface, plc platform.ExecClient, endpointID string) error { var err error log.Printf("[net] Deleting endpoint %v from network %v.", endpointID, nw.Id) @@ -140,7 +141,7 @@ func (nw *network) deleteEndpoint(cli apipaClient, nl netlink.NetlinkInterface, } // Call the platform implementation. - err = nw.deleteEndpointImpl(cli, nl, ep) + err = nw.deleteEndpointImpl(cli, nl, plc, ep) if err != nil { return err } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 49f9fd88e4..312c026e08 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -11,8 +11,11 @@ import ( "strings" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" ) const ( @@ -45,7 +48,7 @@ func ConstructEndpointID(containerID string, _ string, ifName string) (string, s } // newEndpointImpl creates a new endpoint in the network. -func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, epInfo *EndpointInfo) (*endpoint, error) { +func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, plc platform.ExecClient, epInfo *EndpointInfo) (*endpoint, error) { var containerIf *net.Interface var ns *Namespace var ep *endpoint @@ -99,13 +102,14 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, e vlanid, localIP, nl, - ovsctl.NewOvsctl()) + ovsctl.NewOvsctl(), + plc) } else if nw.Mode != opModeTransparent { log.Printf("Bridge client") - epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl) + epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) } else { log.Printf("Transparent client") - epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl) + epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) } // Cleanup on failure. @@ -179,6 +183,15 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, e }() } + if epInfo.IPV6Mode != "" { + // Enable ipv6 setting in container + log.Printf("Enable ipv6 setting in container.") + nuc := networkutils.NewNetworkUtils(nl, plc) + if err = nuc.UpdateIPV6Setting(0); err != nil { + return nil, fmt.Errorf("Enable ipv6 in container failed:%w", err) + } + } + // If a name for the container interface is specified... if epInfo.IfName != "" { if err = epClient.SetupContainerInterfaces(epInfo); err != nil { @@ -218,7 +231,7 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, e } // deleteEndpointImpl deletes an existing endpoint from the network. -func (nw *network) deleteEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, ep *endpoint) error { +func (nw *network) deleteEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, plc platform.ExecClient, ep *endpoint) error { var epClient EndpointClient // Delete the veth pair by deleting one of the peer interfaces. @@ -226,11 +239,11 @@ func (nw *network) deleteEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface // entering the container netns and hence works both for CNI and CNM. if ep.VlanID != 0 { epInfo := ep.getInfo() - epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl()) + epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) } else if nw.Mode != opModeTransparent { - epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl) + epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl, plc) } else { - epClient = NewTransparentEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl) + epClient = NewTransparentEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl, plc) } epClient.DeleteEndpointRules(ep) @@ -243,17 +256,21 @@ func (nw *network) deleteEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface func (ep *endpoint) getInfoImpl(epInfo *EndpointInfo) { } -func addRoutes(nl netlink.NetlinkInterface, interfaceName string, routes []RouteInfo) error { +func addRoutes(nl netlink.NetlinkInterface, netioshim netio.NetIOInterface, interfaceName string, routes []RouteInfo) error { ifIndex := 0 - interfaceIf, _ := net.InterfaceByName(interfaceName) for _, route := range routes { log.Printf("[net] Adding IP route %+v to link %v.", route, interfaceName) if route.DevName != "" { - devIf, _ := net.InterfaceByName(route.DevName) + devIf, _ := netioshim.GetNetworkInterfaceByName(route.DevName) ifIndex = devIf.Index } else { + interfaceIf, err := netioshim.GetNetworkInterfaceByName(interfaceName) + if err != nil { + log.Errorf("Interface not found:%v", err) + return fmt.Errorf("addRoutes failed: %w", err) + } ifIndex = interfaceIf.Index } @@ -284,15 +301,14 @@ func addRoutes(nl netlink.NetlinkInterface, interfaceName string, routes []Route return nil } -func deleteRoutes(nl netlink.NetlinkInterface, interfaceName string, routes []RouteInfo) error { +func deleteRoutes(nl netlink.NetlinkInterface, netioshim netio.NetIOInterface, interfaceName string, routes []RouteInfo) error { ifIndex := 0 - interfaceIf, _ := net.InterfaceByName(interfaceName) for _, route := range routes { log.Printf("[net] Deleting IP route %+v from link %v.", route, interfaceName) if route.DevName != "" { - devIf, _ := net.InterfaceByName(route.DevName) + devIf, _ := netioshim.GetNetworkInterfaceByName(route.DevName) if devIf == nil { log.Printf("[net] Not deleting route. Interface %v doesn't exist", interfaceName) continue @@ -300,6 +316,7 @@ func deleteRoutes(nl netlink.NetlinkInterface, interfaceName string, routes []Ro ifIndex = devIf.Index } else { + interfaceIf, _ := netioshim.GetNetworkInterfaceByName(interfaceName) if interfaceIf == nil { log.Printf("[net] Not deleting route. Interface %v doesn't exist", interfaceName) continue @@ -307,9 +324,13 @@ func deleteRoutes(nl netlink.NetlinkInterface, interfaceName string, routes []Ro ifIndex = interfaceIf.Index } + family := netlink.GetIPAddressFamily(route.Gw) + if route.Gw == nil { + family = netlink.GetIPAddressFamily(route.Dst.IP) + } nlRoute := &netlink.Route{ - Family: netlink.GetIPAddressFamily(route.Gw), + Family: family, Dst: &route.Dst, Gw: route.Gw, LinkIndex: ifIndex, @@ -442,12 +463,12 @@ func (nm *networkManager) updateRoutes(existingEp *EndpointInfo, targetEp *Endpo } - err := deleteRoutes(nm.netlink, existingEp.IfName, tobeDeletedRoutes) + err := deleteRoutes(nm.netlink, &netio.NetIO{}, existingEp.IfName, tobeDeletedRoutes) if err != nil { return err } - err = addRoutes(nm.netlink, existingEp.IfName, tobeAddedRoutes) + err = addRoutes(nm.netlink, &netio.NetIO{}, existingEp.IfName, tobeAddedRoutes) if err != nil { return err } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 09caddde16..ea1d51381e 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -93,7 +93,7 @@ func ConstructEndpointID(containerID string, netNsPath string, ifName string) (s } // newEndpointImpl creates a new endpoint in the network. -func (nw *network) newEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, epInfo *EndpointInfo) (*endpoint, error) { +func (nw *network) newEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, _ platform.ExecClient, epInfo *EndpointInfo) (*endpoint, error) { if useHnsV2, err := UseHnsV2(epInfo.NetNsPath); useHnsV2 { if err != nil { return nil, err @@ -415,7 +415,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( } // deleteEndpointImpl deletes an existing endpoint from the network. -func (nw *network) deleteEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, ep *endpoint) error { +func (nw *network) deleteEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, _ platform.ExecClient, ep *endpoint) error { if useHnsV2, err := UseHnsV2(ep.NetNs); useHnsV2 { if err != nil { return err diff --git a/network/errors.go b/network/errors.go new file mode 100644 index 0000000000..95ab57d2ca --- /dev/null +++ b/network/errors.go @@ -0,0 +1,8 @@ +package network + +import "errors" + +var ( + errSubnetV6NotFound = errors.New("Couldn't find ipv6 subnet in network info") + errV6SnatRuleNotSet = errors.New("ipv6 snat rule not set. Might be VM ipv6 address missing") +) diff --git a/network/manager.go b/network/manager.go index c7a954365c..4ae1829924 100644 --- a/network/manager.go +++ b/network/manager.go @@ -58,6 +58,7 @@ type networkManager struct { ExternalInterfaces map[string]*externalInterface store store.KeyValueStore netlink netlink.NetlinkInterface + plClient platform.ExecClient sync.Mutex } @@ -85,10 +86,11 @@ type NetworkManager interface { } // Creates a new network manager. -func NewNetworkManager(nl netlink.NetlinkInterface) (NetworkManager, error) { +func NewNetworkManager(nl netlink.NetlinkInterface, plc platform.ExecClient) (NetworkManager, error) { nm := &networkManager{ ExternalInterfaces: make(map[string]*externalInterface), netlink: nl, + plClient: plc, } return nm, nil @@ -333,7 +335,7 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn } } - _, err = nw.newEndpoint(cli, nm.netlink, epInfo) + _, err = nw.newEndpoint(cli, nm.netlink, nm.plClient, epInfo) if err != nil { return err } @@ -356,7 +358,7 @@ func (nm *networkManager) DeleteEndpoint(cli apipaClient, networkID string, endp return err } - err = nw.deleteEndpoint(cli, nm.netlink, endpointID) + err = nw.deleteEndpoint(cli, nm.netlink, nm.plClient, endpointID) if err != nil { return err } diff --git a/network/network_linux.go b/network/network_linux.go index d3fdeb0b2c..df7355e235 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -12,8 +12,9 @@ import ( "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" "github.com/Azure/azure-container-networking/platform" "golang.org/x/sys/unix" @@ -83,9 +84,12 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt log.Printf("bridge handleCommonOptions failed with error %s", err.Error()) } case opModeTransparent: - err := nm.handleCommonOptions(extIf.Name, nwInfo) - if err != nil { - log.Printf("transparent handleCommonOptions failed with error %s", err.Error()) + log.Printf("Transparent mode") + if nwInfo.IPV6Mode != "" { + nu := networkutils.NewNetworkUtils(nm.netlink, nm.plClient) + if err := nu.EnableIPV6Forwarding(); err != nil { + return nil, fmt.Errorf("Ipv6 forwarding failed: %w", err) + } } default: return nil, errNetworkModeInvalid @@ -129,9 +133,9 @@ func (nm *networkManager) deleteNetworkImpl(nw *network) error { var networkClient NetworkClient if nw.VlanId != 0 { - networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, ovsctl.NewOvsctl()) + networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, ovsctl.NewOvsctl(), nm.netlink, nm.plClient) } else { - networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, NetworkInfo{}, nm.netlink) + networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, NetworkInfo{}, nm.netlink, nm.plClient) } // Disconnect the interface if this was the last network using it. @@ -236,8 +240,9 @@ func isGreaterOrEqaulUbuntuVersion(versionToMatch int) bool { func readDnsInfo(ifName string) (DNSInfo, error) { var dnsInfo DNSInfo + p := platform.NewExecClient() cmd := fmt.Sprintf("systemd-resolve --status %s", ifName) - out, err := platform.ExecuteCommand(cmd) + out, err := p.ExecuteCommand(cmd) if err != nil { return dnsInfo, err } @@ -325,6 +330,7 @@ func applyDnsConfig(extIf *externalInterface, ifName string) error { setDnsList string err error ) + p := platform.NewExecClient() if extIf != nil { for _, server := range extIf.DNSInfo.Servers { @@ -339,7 +345,7 @@ func applyDnsConfig(extIf *externalInterface, ifName string) error { if setDnsList != "" { cmd := fmt.Sprintf("systemd-resolve --interface=%s%s", ifName, setDnsList) - _, err = platform.ExecuteCommand(cmd) + _, err = p.ExecuteCommand(cmd) if err != nil { return err } @@ -347,7 +353,7 @@ func applyDnsConfig(extIf *externalInterface, ifName string) error { if extIf.DNSInfo.Suffix != "" { cmd := fmt.Sprintf("systemd-resolve --interface=%s --set-domain=%s", ifName, extIf.DNSInfo.Suffix) - _, err = platform.ExecuteCommand(cmd) + _, err = p.ExecuteCommand(cmd) } } @@ -385,9 +391,9 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI opt, _ := nwInfo.Options[genericData].(map[string]interface{}) if opt != nil && opt[VlanIDKey] != nil { - networkClient = NewOVSClient(bridgeName, extIf.Name, ovsctl.NewOvsctl()) + networkClient = NewOVSClient(bridgeName, extIf.Name, ovsctl.NewOvsctl(), nm.netlink, nm.plClient) } else { - networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, *nwInfo, nm.netlink) + networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, *nwInfo, nm.netlink, nm.plClient) } // Check if the bridge already exists. @@ -428,8 +434,9 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI isGreaterOrEqualUbuntu17 := isGreaterOrEqaulUbuntuVersion(ubuntuVersion17) isSystemdResolvedActive := false if isGreaterOrEqualUbuntu17 { + p := platform.NewExecClient() // Don't copy dns servers if systemd-resolved isn't available - if _, cmderr := platform.ExecuteCommand("systemctl status systemd-resolved"); cmderr == nil { + if _, cmderr := p.ExecuteCommand("systemctl status systemd-resolved"); cmderr == nil { isSystemdResolvedActive = true log.Printf("[net] Saving dns config from %v", extIf.Name) if err = saveDnsConfig(extIf); err != nil { @@ -500,12 +507,12 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI if nwInfo.IPV6Mode == IPV6Nat { // adds pod cidr gateway ip to bridge - if err = addIpv6NatGateway(nm.netlink, nwInfo); err != nil { + if err = nm.addIpv6NatGateway(nwInfo); err != nil { log.Errorf("[net] Adding IPv6 Nat Gateway failed:%v", err) return err } - if err = addIpv6SnatRule(extIf, nwInfo); err != nil { + if err = nm.addIpv6SnatRule(extIf, nwInfo); err != nil { log.Errorf("[net] Adding IPv6 Snat Rule failed:%v", err) return err } @@ -593,7 +600,7 @@ func (nm *networkManager) addBridgeRoutes(bridgeName string, routes []RouteInfo) } // Add ipv6 nat gateway IP on bridge -func addIpv6NatGateway(nl netlink.NetlinkInterface, nwInfo *NetworkInfo) error { +func (nm *networkManager) addIpv6NatGateway(nwInfo *NetworkInfo) error { log.Printf("[net] Adding ipv6 nat gateway on azure bridge") for _, subnetInfo := range nwInfo.Subnets { if subnetInfo.Family == platform.AfINET6 { @@ -601,8 +608,8 @@ func addIpv6NatGateway(nl netlink.NetlinkInterface, nwInfo *NetworkInfo) error { IP: subnetInfo.Gateway, Mask: subnetInfo.Prefix.Mask, }} - epc := epcommon.NewEPCommon(nl) - err := epc.AssignIPToInterface(nwInfo.BridgeName, ipAddr) + nuc := networkutils.NewNetworkUtils(nm.netlink, nm.plClient) + err := nuc.AssignIPToInterface(nwInfo.BridgeName, ipAddr) if err != nil { return newErrorNetworkManager(err.Error()) } @@ -613,16 +620,38 @@ func addIpv6NatGateway(nl netlink.NetlinkInterface, nwInfo *NetworkInfo) error { } // snat ipv6 traffic to secondary ipv6 ip before leaving VM -func addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { - log.Printf("[net] Adding ipv6 snat rule") +func (*networkManager) addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error { + var ( + ipv6SnatRuleSet bool + ipv6SubnetPrefix net.IPNet + ) + + for _, subnet := range nwInfo.Subnets { + if subnet.Family == platform.AfINET6 { + ipv6SubnetPrefix = subnet.Prefix + break + } + } + + if len(ipv6SubnetPrefix.IP) == 0 { + return errSubnetV6NotFound + } + for _, ipAddr := range extIf.IPAddresses { if ipAddr.IP.To4() == nil { - if err := epcommon.AddSnatRule("", ipAddr.IP); err != nil { - return err + log.Printf("[net] Adding ipv6 snat rule") + matchSrcPrefix := fmt.Sprintf("-s %s", ipv6SubnetPrefix.String()) + if err := networkutils.AddSnatRule(matchSrcPrefix, ipAddr.IP); err != nil { + return fmt.Errorf("Adding iptable snat rule failed:%w", err) } + ipv6SnatRuleSet = true } } + if !ipv6SnatRuleSet { + return errV6SnatRuleNotSet + } + return nil } @@ -635,14 +664,14 @@ func getNetworkInfoImpl(nwInfo *NetworkInfo, nw *network) { } // AddStaticRoute adds a static route to the interface. -func AddStaticRoute(nl netlink.NetlinkInterface, ip, interfaceName string) error { +func AddStaticRoute(nl netlink.NetlinkInterface, netioshim netio.NetIOInterface, ip, interfaceName string) error { log.Printf("[ovs] Adding %v static route", ip) var routes []RouteInfo _, ipNet, _ := net.ParseCIDR(ip) gwIP := net.ParseIP("0.0.0.0") route := RouteInfo{Dst: *ipNet, Gw: gwIP} routes = append(routes, route) - if err := addRoutes(nl, interfaceName, routes); err != nil { + if err := addRoutes(nl, netioshim, interfaceName, routes); err != nil { if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { log.Printf("addroutes failed with error %v", err) return err diff --git a/network/network_test.go b/network/network_test.go index ec4f691843..73d168ee13 100644 --- a/network/network_test.go +++ b/network/network_test.go @@ -4,6 +4,7 @@ import ( "net" "testing" + "github.com/Azure/azure-container-networking/platform" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -158,6 +159,28 @@ var _ = Describe("Test Network", func() { Expect(nw).To(BeNil()) }) }) + + Context("create new network in transparent mode", func() { + It("Should create new network", func() { + nm := &networkManager{ + ExternalInterfaces: map[string]*externalInterface{}, + plClient: platform.NewMockExecClient(false), + } + nm.ExternalInterfaces["eth0"] = &externalInterface{ + Networks: map[string]*network{}, + } + nwInfo := &NetworkInfo{ + Id: "nw", + MasterIfName: "eth0", + Mode: opModeTransparent, + IPV6Mode: IPV6Nat, + } + nw, err := nm.newNetwork(nwInfo) + Expect(err).To(BeNil()) + Expect(nw).NotTo(BeNil()) + Expect(nw.Id).To(Equal(nwInfo.Id)) + }) + }) }) Describe("Test deleteNetwork", func() { diff --git a/network/network_windows.go b/network/network_windows.go index aa6c606684..8544988804 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -12,7 +12,6 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/policy" - "github.com/Azure/azure-container-networking/platform" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" "github.com/google/uuid" @@ -146,7 +145,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern }() // route entry for pod cidr - if err = appIPV6RouteEntry(nwInfo); err != nil { + if err = nm.appIPV6RouteEntry(nwInfo); err != nil { return nil, err } @@ -173,7 +172,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern return nw, nil } -func appIPV6RouteEntry(nwInfo *NetworkInfo) error { +func (nm *networkManager) appIPV6RouteEntry(nwInfo *NetworkInfo) error { var ( err error out string @@ -192,13 +191,13 @@ func appIPV6RouteEntry(nwInfo *NetworkInfo) error { cmd := fmt.Sprintf(routeCmd, "delete", nwInfo.Subnets[1].Prefix.String(), ifName, ipv6DefaultHop) - if out, err = platform.ExecuteCommand(cmd); err != nil { + if out, err = nm.plClient.ExecuteCommand(cmd); err != nil { log.Printf("[net] Deleting ipv6 route failed: %v:%v", out, err) } cmd = fmt.Sprintf(routeCmd, "add", nwInfo.Subnets[1].Prefix.String(), ifName, ipv6DefaultHop) - if out, err = platform.ExecuteCommand(cmd); err != nil { + if out, err = nm.plClient.ExecuteCommand(cmd); err != nil { log.Printf("[net] Adding ipv6 route failed: %v:%v", out, err) } } diff --git a/network/epcommon/endpoint_common.go b/network/networkutils/networkutils_linux.go similarity index 74% rename from network/epcommon/endpoint_common.go rename to network/networkutils/networkutils_linux.go index af9cd403d0..db9e8d6ca7 100644 --- a/network/epcommon/endpoint_common.go +++ b/network/networkutils/networkutils_linux.go @@ -1,7 +1,7 @@ //go:build linux // +build linux -package epcommon +package networkutils import ( "errors" @@ -37,23 +37,25 @@ const ( acceptRAV6File = "/proc/sys/net/ipv6/conf/%s/accept_ra" ) -var errorEPCommon = errors.New("ErrorEPCommon Error") +var errorNetworkUtils = errors.New("NetworkUtils Error") -func newErrorEPCommon(errStr string) error { - return fmt.Errorf("%w : %s", errorEPCommon, errStr) +func newErrorNetworkUtils(errStr string) error { + return fmt.Errorf("%w : %s", errorNetworkUtils, errStr) } -type EPCommon struct { - netlink netlink.NetlinkInterface +type NetworkUtils struct { + netlink netlink.NetlinkInterface + plClient platform.ExecClient } -func NewEPCommon(nl netlink.NetlinkInterface) EPCommon { - return EPCommon{ - netlink: nl, +func NewNetworkUtils(nl netlink.NetlinkInterface, plClient platform.ExecClient) NetworkUtils { + return NetworkUtils{ + netlink: nl, + plClient: plClient, } } -func (epc EPCommon) CreateEndpoint(hostVethName string, containerVethName string) error { +func (nu NetworkUtils) CreateEndpoint(hostVethName, containerVethName string) error { log.Printf("[net] Creating veth pair %v %v.", hostVethName, containerVethName) link := netlink.VEthLink{ @@ -64,66 +66,66 @@ func (epc EPCommon) CreateEndpoint(hostVethName string, containerVethName string PeerName: containerVethName, } - err := epc.netlink.AddLink(&link) + err := nu.netlink.AddLink(&link) if err != nil { log.Printf("[net] Failed to create veth pair, err:%v.", err) - return newErrorEPCommon(err.Error()) + return newErrorNetworkUtils(err.Error()) } log.Printf("[net] Setting link %v state up.", hostVethName) - err = epc.netlink.SetLinkState(hostVethName, true) + err = nu.netlink.SetLinkState(hostVethName, true) if err != nil { - return newErrorEPCommon(err.Error()) + return newErrorNetworkUtils(err.Error()) } - if err := DisableRAForInterface(hostVethName); err != nil { - return newErrorEPCommon(err.Error()) + if err := nu.DisableRAForInterface(hostVethName); err != nil { + return newErrorNetworkUtils(err.Error()) } return nil } -func (epc EPCommon) SetupContainerInterface(containerVethName string, targetIfName string) error { +func (nu NetworkUtils) SetupContainerInterface(containerVethName, targetIfName string) error { // Interface needs to be down before renaming. log.Printf("[net] Setting link %v state down.", containerVethName) - if err := epc.netlink.SetLinkState(containerVethName, false); err != nil { - return newErrorEPCommon(err.Error()) + if err := nu.netlink.SetLinkState(containerVethName, false); err != nil { + return newErrorNetworkUtils(err.Error()) } // Rename the container interface. log.Printf("[net] Setting link %v name %v.", containerVethName, targetIfName) - if err := epc.netlink.SetLinkName(containerVethName, targetIfName); err != nil { - return newErrorEPCommon(err.Error()) + if err := nu.netlink.SetLinkName(containerVethName, targetIfName); err != nil { + return newErrorNetworkUtils(err.Error()) } - if err := DisableRAForInterface(targetIfName); err != nil { - return newErrorEPCommon(err.Error()) + if err := nu.DisableRAForInterface(targetIfName); err != nil { + return newErrorNetworkUtils(err.Error()) } // Bring the interface back up. log.Printf("[net] Setting link %v state up.", targetIfName) - err := epc.netlink.SetLinkState(targetIfName, true) + err := nu.netlink.SetLinkState(targetIfName, true) if err != nil { - return newErrorEPCommon(err.Error()) + return newErrorNetworkUtils(err.Error()) } return nil } -func (epc EPCommon) AssignIPToInterface(interfaceName string, ipAddresses []net.IPNet) error { +func (nu NetworkUtils) AssignIPToInterface(interfaceName string, ipAddresses []net.IPNet) error { var err error // Assign IP address to container network interface. for i, ipAddr := range ipAddresses { log.Printf("[net] Adding IP address %v to link %v.", ipAddr.String(), interfaceName) - err = epc.netlink.AddIPAddress(interfaceName, ipAddr.IP, &ipAddresses[i]) + err = nu.netlink.AddIPAddress(interfaceName, ipAddr.IP, &ipAddresses[i]) if err != nil { - return newErrorEPCommon(err.Error()) + return newErrorNetworkUtils(err.Error()) } } return nil } -func addOrDeleteFilterRule(bridgeName string, action string, ipAddress string, chainName string, target string) error { +func addOrDeleteFilterRule(bridgeName, action, ipAddress, chainName, target string) error { var err error option := "i" @@ -169,7 +171,7 @@ func AllowIPAddresses(bridgeName string, skipAddresses []string, action string) return nil } -func BlockIPAddresses(bridgeName string, action string) error { +func BlockIPAddresses(bridgeName, action string) error { privateIPAddresses := getPrivateIPSpace() chains := getFilterChains() target := getFilterchainTarget() @@ -194,11 +196,11 @@ func BlockIPAddresses(bridgeName string, action string) error { } // This fucntion enables ip forwarding in VM and allow forwarding packets from the interface -func EnableIPForwarding(ifName string) error { +func (nu NetworkUtils) EnableIPForwarding(ifName string) error { // Enable ip forwading on linux vm. // sysctl -w net.ipv4.ip_forward=1 cmd := fmt.Sprint(enableIPForwardCmd) - _, err := platform.ExecuteCommand(cmd) + _, err := nu.plClient.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Enable ipforwarding failed with: %v", err) return err @@ -213,9 +215,9 @@ func EnableIPForwarding(ifName string) error { return nil } -func EnableIPV6Forwarding() error { +func (nu NetworkUtils) EnableIPV6Forwarding() error { cmd := fmt.Sprint(enableIPV6ForwardCmd) - _, err := platform.ExecuteCommand(cmd) + _, err := nu.plClient.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Enable ipv6 forwarding failed with: %v", err) return err @@ -225,10 +227,10 @@ func EnableIPV6Forwarding() error { } // This functions enables/disables ipv6 setting based on enable parameter passed. -func UpdateIPV6Setting(disable int) error { +func (nu NetworkUtils) UpdateIPV6Setting(disable int) error { // sysctl -w net.ipv6.conf.all.disable_ipv6=0/1 cmd := fmt.Sprintf(toggleIPV6Cmd, disable) - _, err := platform.ExecuteCommand(cmd) + _, err := nu.plClient.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Update IPV6 Setting failed with: %v", err) } @@ -247,7 +249,7 @@ func AddSnatRule(match string, ip net.IP) error { return iptables.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target) } -func DisableRAForInterface(ifName string) error { +func (nu NetworkUtils) DisableRAForInterface(ifName string) error { raFilePath := fmt.Sprintf(acceptRAV6File, ifName) exist, err := platform.CheckIfFileExists(raFilePath) if !exist { @@ -256,7 +258,7 @@ func DisableRAForInterface(ifName string) error { } cmd := fmt.Sprintf(disableRACmd, ifName) - out, err := platform.ExecuteCommand(cmd) + out, err := nu.plClient.ExecuteCommand(cmd) if err != nil { log.Errorf("[net] Diabling ra failed with err: %v out: %v", err, out) } diff --git a/network/ovs_endpoint_infraroute_linux.go b/network/ovs_endpoint_infraroute_linux.go index 91a2c50a06..829a58ee4c 100644 --- a/network/ovs_endpoint_infraroute_linux.go +++ b/network/ovs_endpoint_infraroute_linux.go @@ -12,7 +12,7 @@ func NewInfraVnetClient(client *OVSEndpointClient, epID string) { hostIfName := fmt.Sprintf("%s%s", infraVethInterfacePrefix, epID) contIfName := fmt.Sprintf("%s%s-2", infraVethInterfacePrefix, epID) - client.infraVnetClient = ovsinfravnet.NewInfraVnetClient(hostIfName, contIfName, client.netlink) + client.infraVnetClient = ovsinfravnet.NewInfraVnetClient(hostIfName, contIfName, client.netlink, client.plClient) } } diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index 8952cd80df..5048bfc54a 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -3,7 +3,7 @@ package network import ( "fmt" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/ovssnat" ) @@ -20,6 +20,7 @@ func NewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP strin epInfo.DNS.Servers, client.netlink, client.ovsctlClient, + client.plClient, ) } } @@ -47,11 +48,12 @@ func AddSnatEndpointRules(client *OVSEndpointClient) error { } // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge - if err := AddStaticRoute(client.netlink, ovssnat.ImdsIP, client.bridgeName); err != nil { + if err := AddStaticRoute(client.netlink, client.netioshim, ovssnat.ImdsIP, client.bridgeName); err != nil { return err } - if err := epcommon.EnableIPForwarding(ovssnat.SnatBridgeName); err != nil { + nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + if err := nuc.EnableIPForwarding(ovssnat.SnatBridgeName); err != nil { return err } diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 4653fc73f0..54054f2572 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -7,11 +7,13 @@ import ( "net" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/ovsinfravnet" "github.com/Azure/azure-container-networking/network/ovssnat" "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" ) type OVSEndpointClient struct { @@ -30,7 +32,9 @@ type OVSEndpointClient struct { allowInboundFromNCToHost bool enableSnatForDns bool netlink netlink.NetlinkInterface + netioshim netio.NetIOInterface ovsctlClient ovsctl.OvsInterface + plClient platform.ExecClient } const ( @@ -46,7 +50,8 @@ func NewOVSEndpointClient( vlanid int, localIP string, nl netlink.NetlinkInterface, - ovs ovsctl.OvsInterface) *OVSEndpointClient { + ovs ovsctl.OvsInterface, + plc platform.ExecClient) *OVSEndpointClient { client := &OVSEndpointClient{ bridgeName: nw.extIf.BridgeName, @@ -62,6 +67,8 @@ func NewOVSEndpointClient( enableSnatForDns: epInfo.EnableSnatForDns, netlink: nl, ovsctlClient: ovs, + plClient: plc, + netioshim: &netio.NetIO{}, } NewInfraVnetClient(client, epInfo.Id[:7]) @@ -71,7 +78,7 @@ func NewOVSEndpointClient( } func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) + epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) if err := epc.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { return err } @@ -195,8 +202,8 @@ func (client *OVSEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo } func (client *OVSEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + if err := nuc.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return err } @@ -210,8 +217,8 @@ func (client *OVSEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) } func (client *OVSEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + if err := nuc.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return err } @@ -223,7 +230,7 @@ func (client *OVSEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *E return err } - return addRoutes(client.netlink, client.containerVethName, epInfo.Routes) + return addRoutes(client.netlink, client.netioshim, client.containerVethName, epInfo.Routes) } func (client *OVSEndpointClient) DeleteEndpoints(ep *endpoint) error { diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index 0703c71f82..bd088b0678 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -11,8 +11,10 @@ import ( "strings" "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" ) var errorOVSNetworkClient = errors.New("OVSNetworkClient Error") @@ -25,6 +27,9 @@ type OVSNetworkClient struct { bridgeName string hostInterfaceName string ovsctlClient ovsctl.OvsInterface + netlink netlink.NetlinkInterface + plClient platform.ExecClient + nuClient networkutils.NetworkUtils } const ( @@ -67,11 +72,15 @@ func (client *OVSNetworkClient) AddRoutes(nwInfo *NetworkInfo, interfaceName str return nil } -func NewOVSClient(bridgeName, hostInterfaceName string, ovsctlClient ovsctl.OvsInterface) *OVSNetworkClient { +func NewOVSClient(bridgeName, hostInterfaceName string, ovsctlClient ovsctl.OvsInterface, + nl netlink.NetlinkInterface, plc platform.ExecClient) *OVSNetworkClient { ovsClient := &OVSNetworkClient{ bridgeName: bridgeName, hostInterfaceName: hostInterfaceName, ovsctlClient: ovsctlClient, + netlink: nl, + plClient: plc, + nuClient: networkutils.NewNetworkUtils(nl, plc), } return ovsClient @@ -90,7 +99,7 @@ func (client *OVSNetworkClient) CreateBridge() error { } }() - if err := epcommon.DisableRAForInterface(client.bridgeName); err != nil { + if err := client.nuClient.DisableRAForInterface(client.bridgeName); err != nil { return err } diff --git a/network/ovs_networkclient_linux_test.go b/network/ovs_networkclient_linux_test.go index 146005e80a..924468b474 100644 --- a/network/ovs_networkclient_linux_test.go +++ b/network/ovs_networkclient_linux_test.go @@ -4,7 +4,9 @@ import ( "os" "testing" + "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" ) const ( @@ -18,7 +20,8 @@ func TestMain(m *testing.M) { func TestAddRoutes(t *testing.T) { ovsctlClient := ovsctl.NewMockOvsctl(false, "", "") - ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient) + ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient, + netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false)) if err := ovsClient.AddRoutes(nil, ""); err != nil { t.Errorf("Add routes failed") @@ -37,7 +40,8 @@ func TestCreateBridge(t *testing.T) { t.Errorf("Unable to write to file %v: %v", ovsConfigFile, err) } - ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient) + ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient, + netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false)) if err := ovsClient.CreateBridge(); err != nil { t.Errorf("Error creating OVS bridge: %v", err) } @@ -48,7 +52,8 @@ func TestCreateBridge(t *testing.T) { func TestDeleteBridge(t *testing.T) { ovsctlClient := ovsctl.NewMockOvsctl(false, "", "") - ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient) + ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient, + netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false)) if err := ovsClient.DeleteBridge(); err != nil { t.Errorf("Error deleting the OVS bridge: %v", err) } @@ -61,7 +66,8 @@ func TestAddL2Rules(t *testing.T) { MacAddress: []byte("2C:54:91:88:C9:E3"), } - ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient) + ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient, + netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false)) if err := ovsClient.AddL2Rules(&extIf); err != nil { t.Errorf("Unable to add L2 rules: %v", err) } @@ -74,14 +80,16 @@ func TestDeleteL2Rules(t *testing.T) { MacAddress: []byte("2C:54:91:88:C9:E3"), } - ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient) + ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient, + netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false)) ovsClient.DeleteL2Rules(&extIf) } func TestSetBridgeMasterToHostInterface(t *testing.T) { ovsctlClient := ovsctl.NewMockOvsctl(false, "", "") - ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient) + ovsClient := NewOVSClient(bridgeName, hostIntf, ovsctlClient, + netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false)) if err := ovsClient.SetBridgeMasterToHostInterface(); err != nil { t.Errorf("Unable to set bridge master to host intf: %v", err) } diff --git a/network/ovsinfravnet/infravnet.go b/network/ovsinfravnet/infravnet.go index 5a9df203f0..0934718642 100644 --- a/network/ovsinfravnet/infravnet.go +++ b/network/ovsinfravnet/infravnet.go @@ -10,8 +10,9 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" ) const ( @@ -29,13 +30,15 @@ type OVSInfraVnetClient struct { ContainerInfraVethName string containerInfraMac string netlink netlink.NetlinkInterface + plClient platform.ExecClient } -func NewInfraVnetClient(hostIfName, contIfName string, nl netlink.NetlinkInterface) OVSInfraVnetClient { +func NewInfraVnetClient(hostIfName, contIfName string, nl netlink.NetlinkInterface, plc platform.ExecClient) OVSInfraVnetClient { infraVnetClient := OVSInfraVnetClient{ hostInfraVethName: hostIfName, ContainerInfraVethName: contIfName, netlink: nl, + plClient: plc, } log.Printf("Initialize new infravnet client %+v", infraVnetClient) @@ -45,7 +48,7 @@ func NewInfraVnetClient(hostIfName, contIfName string, nl netlink.NetlinkInterfa func (client *OVSInfraVnetClient) CreateInfraVnetEndpoint(bridgeName string) error { ovs := ovsctl.NewOvsctl() - epc := epcommon.NewEPCommon(client.netlink) + epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) if err := epc.CreateEndpoint(client.hostInfraVethName, client.ContainerInfraVethName); err != nil { log.Printf("Creating infraep failed with error %v", err) return err @@ -107,7 +110,7 @@ func (client *OVSInfraVnetClient) MoveInfraEndpointToContainerNS(netnsPath strin } func (client *OVSInfraVnetClient) SetupInfraVnetContainerInterface() error { - epc := epcommon.NewEPCommon(client.netlink) + epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) if err := epc.SetupContainerInterface(client.ContainerInfraVethName, azureInfraIfName); err != nil { return newErrorOVSInfraVnetClient(err.Error()) } diff --git a/network/ovssnat/ovssnat.go b/network/ovssnat/ovssnat.go index e9df1a7f63..60c531fc67 100644 --- a/network/ovssnat/ovssnat.go +++ b/network/ovssnat/ovssnat.go @@ -13,7 +13,7 @@ import ( "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" "github.com/Azure/azure-container-networking/platform" ) @@ -45,6 +45,7 @@ type OVSSnatClient struct { SkipAddressesFromBlock []string netlink netlink.NetlinkInterface ovsctlClient ovsctl.OvsInterface + plClient platform.ExecClient } func NewSnatClient(hostIfName string, @@ -55,6 +56,7 @@ func NewSnatClient(hostIfName string, skipAddressesFromBlock []string, nl netlink.NetlinkInterface, ovsctlClient ovsctl.OvsInterface, + plClient platform.ExecClient, ) OVSSnatClient { log.Printf("Initialize new snat client") snatClient := OVSSnatClient{ @@ -65,6 +67,7 @@ func NewSnatClient(hostIfName string, hostPrimaryMac: hostPrimaryMac, netlink: nl, ovsctlClient: ovsctlClient, + plClient: plClient, } snatClient.SkipAddressesFromBlock = append(snatClient.SkipAddressesFromBlock, skipAddressesFromBlock...) @@ -93,7 +96,7 @@ func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { return err } - epc := epcommon.NewEPCommon(client.netlink) + epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) // Create veth pair to tie one end to container and other end to linux bridge if err := epc.CreateEndpoint(client.hostSnatVethName, client.containerSnatVethName); err != nil { log.Printf("Creating Snat Endpoint failed with error %v", err) @@ -109,7 +112,7 @@ func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { // AllowIPAddressesOnSnatBridge adds iptables rules that allows only specific Private IPs via linux bridge func (client *OVSSnatClient) AllowIPAddressesOnSnatBridge() error { - if err := epcommon.AllowIPAddresses(SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { + if err := networkutils.AllowIPAddresses(SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { log.Printf("AllowIPAddresses failed with error %v", err) return newErrorOVSSnatClient(err.Error()) } @@ -119,7 +122,7 @@ func (client *OVSSnatClient) AllowIPAddressesOnSnatBridge() error { // BlockIPAddressesOnSnatBridge adds iptables rules that blocks all private IPs flowing via linux bridge func (client *OVSSnatClient) BlockIPAddressesOnSnatBridge() error { - if err := epcommon.BlockIPAddresses(SnatBridgeName, iptables.Append); err != nil { + if err := networkutils.BlockIPAddresses(SnatBridgeName, iptables.Append); err != nil { log.Printf("AllowIPAddresses failed with error %v", err) return newErrorOVSSnatClient(err.Error()) } @@ -143,7 +146,7 @@ func (client *OVSSnatClient) MoveSnatEndpointToContainerNS(netnsPath string, nsI Configure Routes and setup name for container veth **/ func (client *OVSSnatClient) SetupSnatContainerInterface() error { - epc := epcommon.NewEPCommon(client.netlink) + epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) if err := epc.SetupContainerInterface(client.containerSnatVethName, azureSnatIfName); err != nil { return newErrorOVSSnatClient(err.Error()) } @@ -404,7 +407,9 @@ func (client *OVSSnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMa return nil } - if err := epcommon.DisableRAForInterface(SnatBridgeName); err != nil { + nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + //nolint + if err = nuc.DisableRAForInterface(SnatBridgeName); err != nil { return err } @@ -422,11 +427,13 @@ func (client *OVSSnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMa return err } - if err := epcommon.DisableRAForInterface(azureSnatVeth0); err != nil { + //nolint + if err = nuc.DisableRAForInterface(azureSnatVeth0); err != nil { return err } - if err := epcommon.DisableRAForInterface(azureSnatVeth1); err != nil { + //nolint + if err = nuc.DisableRAForInterface(azureSnatVeth1); err != nil { return err } @@ -439,23 +446,23 @@ func (client *OVSSnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMa return newErrorOVSSnatClient(err.Error()) } - if err := client.netlink.SetLinkState(SnatBridgeName, true); err != nil { + if err = client.netlink.SetLinkState(SnatBridgeName, true); err != nil { return newErrorOVSSnatClient(err.Error()) } - if err := client.netlink.SetLinkState(azureSnatVeth0, true); err != nil { + if err = client.netlink.SetLinkState(azureSnatVeth0, true); err != nil { return newErrorOVSSnatClient(err.Error()) } - if err := client.netlink.SetLinkMaster(azureSnatVeth0, SnatBridgeName); err != nil { + if err = client.netlink.SetLinkMaster(azureSnatVeth0, SnatBridgeName); err != nil { return newErrorOVSSnatClient(err.Error()) } - if err := client.netlink.SetLinkState(azureSnatVeth1, true); err != nil { + if err = client.netlink.SetLinkState(azureSnatVeth1, true); err != nil { return newErrorOVSSnatClient(err.Error()) } - if err := client.ovsctlClient.AddPortOnOVSBridge(azureSnatVeth1, mainInterface, 0); err != nil { + if err = client.ovsctlClient.AddPortOnOVSBridge(azureSnatVeth1, mainInterface, 0); err != nil { return newErrorOVSSnatClient(err.Error()) } @@ -475,7 +482,7 @@ func (client *OVSSnatClient) addMasqueradeRule(snatBridgeIPWithPrefix string) er Drop all vlan traffic on linux bridge **/ func (client *OVSSnatClient) addVlanDropRule() error { - out, err := platform.ExecuteCommand(l2PreroutingEntries) + out, err := client.plClient.ExecuteCommand(l2PreroutingEntries) if err != nil { log.Printf("Error while listing ebtable rules %v", err) return err @@ -488,6 +495,6 @@ func (client *OVSSnatClient) addVlanDropRule() error { } log.Printf("Adding ebtable rule to drop vlan traffic on snat bridge %v", vlanDropAddRule) - _, err = platform.ExecuteCommand(vlanDropAddRule) + _, err = client.plClient.ExecuteCommand(vlanDropAddRule) return err } diff --git a/network/transparent_endpoint_linux_test.go b/network/transparent_endpoint_linux_test.go index c36548975a..50eaadfce5 100644 --- a/network/transparent_endpoint_linux_test.go +++ b/network/transparent_endpoint_linux_test.go @@ -3,14 +3,25 @@ package network import ( + "net" "testing" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/platform" "github.com/stretchr/testify/require" ) -func TestAddEndpoints(t *testing.T) { +const ( + subnetv4Mask = 24 + subnetv6Mask = 64 +) + +func TestTransAddEndpoints(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + tests := []struct { name string client *TransparentEndpointClient @@ -25,6 +36,8 @@ func TestAddEndpoints(t *testing.T) { hostVethName: "azvhost", containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, @@ -37,6 +50,8 @@ func TestAddEndpoints(t *testing.T) { hostVethName: "azvhost", containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, @@ -50,6 +65,8 @@ func TestAddEndpoints(t *testing.T) { hostVethName: "azvhost", containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, @@ -62,6 +79,8 @@ func TestAddEndpoints(t *testing.T) { hostVethName: "azvhost", containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 2), }, epInfo: &EndpointInfo{}, @@ -75,6 +94,8 @@ func TestAddEndpoints(t *testing.T) { hostVethName: "azvhost", containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 3), }, epInfo: &EndpointInfo{}, @@ -88,6 +109,8 @@ func TestAddEndpoints(t *testing.T) { hostVethName: "azvhost", containerVethName: "azvcontainer", netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(true, 4), }, epInfo: &EndpointInfo{}, @@ -109,3 +132,259 @@ func TestAddEndpoints(t *testing.T) { }) } } + +func TestTransAddEndpointsRules(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + tests := []struct { + name string + client *TransparentEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Add endpoint rules happy path", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: false, + }, + { + name: "Add endpoint rules Dualstack happy path", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPV6Mode: IPV6Nat, + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4FullMask), + }, + { + IP: net.ParseIP("fc00::4"), + Mask: net.CIDRMask(subnetv6Mask, ipv6FullMask), + }, + }, + }, + wantErr: false, + }, + { + name: "Add endpoint rules fail", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(true, "addroute fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: "addroute fail", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.AddEndpointRules(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestTransDeleteEndpointsRules(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + tests := []struct { + name string + client *TransparentEndpointClient + ep *endpoint + }{ + { + name: "Delete endpoint rules happy path", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + ep: &endpoint{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + { + IP: net.ParseIP("fc00::4"), + Mask: net.CIDRMask(subnetv6Mask, ipv6FullMask), + }, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + tt.client.DeleteEndpointRules(tt.ep) + }) + } +} + +func TestTransConfigureContainerInterfacesAndRoutes(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + tests := []struct { + name string + client *TransparentEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Configure Interface and routes happy path", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: false, + }, + { + name: "Configure Interface and routes dualstack happy path", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + { + IP: net.ParseIP("fc00::4"), + Mask: net.CIDRMask(subnetv6Mask, ipv6FullMask), + }, + }, + }, + wantErr: false, + }, + { + name: "Configure Interface and routes assign ip fail", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: "netlink fail", + }, + { + name: "Configure Interface and routes add routes fail", + client: &TransparentEndpointClient{ + hostPrimaryIfName: "eth0", + hostVethName: "azvhost", + containerVethName: "azvcontainer", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 2), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: netio.ErrMockNetIOFail.Error(), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.ConfigureContainerInterfacesAndRoutes(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 7f9af6641b..17a6d5677e 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -8,7 +8,7 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" ) @@ -16,6 +16,12 @@ const ( virtualGwIPString = "169.254.1.1/32" defaultGwCidr = "0.0.0.0/0" defaultGw = "0.0.0.0" + virtualv6GwString = "fe80::1234:5678:9abc/128" + defaultv6Cidr = "::/0" + ipv4Bits = 32 + ipv6Bits = 128 + ipv4FullMask = 32 + ipv6FullMask = 128 ) var errorTransparentEndpointClient = errors.New("TransparentEndpointClient Error") @@ -35,6 +41,8 @@ type TransparentEndpointClient struct { mode string netlink netlink.NetlinkInterface netioshim netio.NetIOInterface + plClient platform.ExecClient + netUtilsClient networkutils.NetworkUtils } func NewTransparentEndpointClient( @@ -43,6 +51,7 @@ func NewTransparentEndpointClient( containerVethName string, mode string, nl netlink.NetlinkInterface, + plc platform.ExecClient, ) *TransparentEndpointClient { client := &TransparentEndpointClient{ @@ -54,14 +63,16 @@ func NewTransparentEndpointClient( mode: mode, netlink: nl, netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), } return client } -func setArpProxy(ifName string) error { +func (client *TransparentEndpointClient) setArpProxy(ifName string) error { cmd := fmt.Sprintf("echo 1 > /proc/sys/net/ipv4/conf/%v/proxy_arp", ifName) - _, err := platform.ExecuteCommand(cmd) + _, err := client.plClient.ExecuteCommand(cmd) return err } @@ -79,8 +90,7 @@ func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) erro return newErrorTransparentEndpointClient(err.Error()) } - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { + if err = client.netUtilsClient.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { return newErrorTransparentEndpointClient(err.Error()) } @@ -124,18 +134,26 @@ func (client *TransparentEndpointClient) AddEndpointRules(epInfo *EndpointInfo) // ip route add dev // This route is needed for incoming packets to pod to route via hostveth for _, ipAddr := range epInfo.IPAddresses { - var routeInfo RouteInfo - ipNet := net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)} + var ( + routeInfo RouteInfo + ipNet net.IPNet + ) + + if ipAddr.IP.To4() != nil { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} + } else { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} + } log.Printf("[net] Adding route for the ip %v", ipNet.String()) routeInfo.Dst = ipNet routeInfoList = append(routeInfoList, routeInfo) - if err := addRoutes(client.netlink, client.hostVethName, routeInfoList); err != nil { + if err := addRoutes(client.netlink, client.netioshim, client.hostVethName, routeInfoList); err != nil { return newErrorTransparentEndpointClient(err.Error()) } } log.Printf("calling setArpProxy for %v", client.hostVethName) - if err := setArpProxy(client.hostVethName); err != nil { + if err := client.setArpProxy(client.hostVethName); err != nil { log.Printf("setArpProxy failed with: %v", err) return err } @@ -144,19 +162,24 @@ func (client *TransparentEndpointClient) AddEndpointRules(epInfo *EndpointInfo) } func (client *TransparentEndpointClient) DeleteEndpointRules(ep *endpoint) { - var routeInfoList []RouteInfo - var err error // ip route del dev // Deleting the route set up for routing the incoming packets to pod for _, ipAddr := range ep.IPAddresses { - var routeInfo RouteInfo - ipNet := net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)} + var ( + routeInfo RouteInfo + ipNet net.IPNet + ) + + if ipAddr.IP.To4() != nil { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} + } else { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} + } + log.Printf("[net] Deleting route for the ip %v", ipNet.String()) routeInfo.Dst = ipNet - routeInfoList = append(routeInfoList, routeInfo) - err = deleteRoutes(client.netlink, client.hostVethName, routeInfoList) - if err != nil { - log.Printf("[net] Failed to delete route for the ip %v: %v", ipNet.String(), err) + if err := deleteRoutes(client.netlink, client.netioshim, client.hostVethName, []RouteInfo{routeInfo}); err != nil { + log.Printf("[net] Failed to delete route on VM for the ip %v: %v", ipNet.String(), err) } } } @@ -172,8 +195,7 @@ func (client *TransparentEndpointClient) MoveEndpointsToContainerNS(epInfo *Endp } func (client *TransparentEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return err } @@ -183,8 +205,7 @@ func (client *TransparentEndpointClient) SetupContainerInterfaces(epInfo *Endpoi } func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - epc := epcommon.NewEPCommon(client.netlink) - if err := epc.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorTransparentEndpointClient(err.Error()) } @@ -196,7 +217,7 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e Scope: netlink.RT_SCOPE_LINK, Protocol: netlink.RTPROT_KERNEL, } - if err := deleteRoutes(client.netlink, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + if err := deleteRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { return newErrorTransparentEndpointClient(err.Error()) } } @@ -208,7 +229,7 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e Dst: *virtualGwNet, Scope: netlink.RT_SCOPE_LINK, } - if err := addRoutes(client.netlink, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { return newErrorTransparentEndpointClient(err.Error()) } @@ -219,16 +240,65 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e Dst: dstIP, Gw: virtualGwIP, } - if err := addRoutes(client.netlink, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { return err } // arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac - log.Printf("[net] Adding static arp for IP address %v and MAC %v in Container namespace", virtualGwNet.String(), client.hostVethMac) - err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, virtualGwNet.IP, client.hostVethMac, false) - if err != nil { - return newErrorTransparentEndpointClient(err.Error()) + log.Printf("[net] Adding static arp for IP address %v and MAC %v in Container namespace", + virtualGwNet.String(), client.hostVethMac) + if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, + client.containerVethName, + virtualGwNet.IP, + client.hostVethMac, + false); err != nil { + return fmt.Errorf("Adding arp in container failed: %w", err) + } + + if epInfo.IPV6Mode != "" { + if err := client.setupIPV6Routes(); err != nil { + return err + } + } + + if epInfo.IPV6Mode != "" { + return client.setIPV6NeighEntry() + } + + return nil +} + +func (client *TransparentEndpointClient) setupIPV6Routes() error { + log.Printf("Setting up ipv6 routes in container") + + // add route for virtualgwip + // ip -6 route add fe80::1234:5678:9abc/128 dev eth0 + virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualv6GwString) + gwRoute := RouteInfo{ + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, } + + // ip -6 route add default via fe80::1234:5678:9abc dev eth0 + _, defaultIPNet, _ := net.ParseCIDR(defaultv6Cidr) + log.Printf("defaultv6ipnet :%+v", defaultIPNet) + defaultRoute := RouteInfo{ + Dst: *defaultIPNet, + Gw: virtualGwIP, + } + + return addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{gwRoute, defaultRoute}) +} + +func (client *TransparentEndpointClient) setIPV6NeighEntry() error { + log.Printf("[net] Add v6 neigh entry for default gw ip") + hostGwIP, _, _ := net.ParseCIDR(virtualv6GwString) + if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName, + hostGwIP, client.hostVethMac, false); err != nil { + log.Printf("Failed setting neigh entry in container: %+v", err) + return fmt.Errorf("Failed setting neigh entry in container: %w", err) + } + return nil } diff --git a/ovsctl/ovsctl.go b/ovsctl/ovsctl.go index 33f0a64e2f..59f9083def 100644 --- a/ovsctl/ovsctl.go +++ b/ovsctl/ovsctl.go @@ -50,17 +50,19 @@ type OvsInterface interface { DeletePortFromOVS(bridgeName string, interfaceName string) error } -type Ovsctl struct{} +type Ovsctl struct { + execcli platform.ExecClient +} func NewOvsctl() Ovsctl { - return Ovsctl{} + return Ovsctl{execcli: platform.NewExecClient()} } -func (Ovsctl) CreateOVSBridge(bridgeName string) error { +func (o Ovsctl) CreateOVSBridge(bridgeName string) error { log.Printf("[ovs] Creating OVS Bridge %v", bridgeName) ovsCreateCmd := fmt.Sprintf("ovs-vsctl add-br %s", bridgeName) - _, err := platform.ExecuteCommand(ovsCreateCmd) + _, err := o.execcli.ExecuteCommand(ovsCreateCmd) if err != nil { log.Printf("[ovs] Error while creating OVS bridge %v", err) return newErrorOvsctl(err.Error()) @@ -69,11 +71,11 @@ func (Ovsctl) CreateOVSBridge(bridgeName string) error { return nil } -func (Ovsctl) DeleteOVSBridge(bridgeName string) error { +func (o Ovsctl) DeleteOVSBridge(bridgeName string) error { log.Printf("[ovs] Deleting OVS Bridge %v", bridgeName) ovsCreateCmd := fmt.Sprintf("ovs-vsctl del-br %s", bridgeName) - _, err := platform.ExecuteCommand(ovsCreateCmd) + _, err := o.execcli.ExecuteCommand(ovsCreateCmd) if err != nil { log.Printf("[ovs] Error while deleting OVS bridge %v", err) return newErrorOvsctl(err.Error()) @@ -82,9 +84,9 @@ func (Ovsctl) DeleteOVSBridge(bridgeName string) error { return nil } -func (Ovsctl) AddPortOnOVSBridge(hostIfName string, bridgeName string, vlanID int) error { +func (o Ovsctl) AddPortOnOVSBridge(hostIfName, bridgeName string, vlanID int) error { cmd := fmt.Sprintf("ovs-vsctl add-port %s %s", bridgeName, hostIfName) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Error while setting OVS as master to primary interface %v", err) return newErrorOvsctl(err.Error()) @@ -93,9 +95,9 @@ func (Ovsctl) AddPortOnOVSBridge(hostIfName string, bridgeName string, vlanID in return nil } -func (Ovsctl) GetOVSPortNumber(interfaceName string) (string, error) { +func (o Ovsctl) GetOVSPortNumber(interfaceName string) (string, error) { cmd := fmt.Sprintf("ovs-vsctl get Interface %s ofport", interfaceName) - ofport, err := platform.ExecuteCommand(cmd) + ofport, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Get ofport failed with error %v", err) return "", newErrorOvsctl(err.Error()) @@ -104,9 +106,9 @@ func (Ovsctl) GetOVSPortNumber(interfaceName string) (string, error) { return strings.Trim(ofport, "\n"), nil } -func (Ovsctl) AddVMIpAcceptRule(bridgeName string, primaryIP string, mac string) error { +func (o Ovsctl) AddVMIpAcceptRule(bridgeName, primaryIP, mac string) error { cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_dst=%s,priority=%d,actions=normal", bridgeName, primaryIP, mac, high) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding SNAT rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -115,10 +117,10 @@ func (Ovsctl) AddVMIpAcceptRule(bridgeName string, primaryIP string, mac string) return nil } -func (Ovsctl) AddArpSnatRule(bridgeName string, mac string, macHex string, ofport string) error { +func (o Ovsctl) AddArpSnatRule(bridgeName, mac, macHex, ofport string) error { cmd := fmt.Sprintf(`ovs-ofctl add-flow %v table=1,priority=%d,arp,arp_op=1,actions='mod_dl_src:%s, load:0x%s->NXM_NX_ARP_SHA[],output:%s'`, bridgeName, low, mac, macHex, ofport) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP SNAT rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -128,7 +130,7 @@ func (Ovsctl) AddArpSnatRule(bridgeName string, mac string, macHex string, ofpor } // IP SNAT Rule - Change src mac to VM Mac for packets coming from container host veth port. -func (Ovsctl) AddIPSnatRule(bridgeName string, ip net.IP, vlanID int, port string, mac string, outport string) error { +func (o Ovsctl) AddIPSnatRule(bridgeName string, ip net.IP, vlanID int, port, mac, outport string) error { var cmd string if outport == "" { outport = "normal" @@ -144,7 +146,7 @@ func (Ovsctl) AddIPSnatRule(bridgeName string, ip net.IP, vlanID int, port strin cmd = fmt.Sprintf("%s,strip_vlan,%v", commonPrefix, outport) } - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding IP SNAT rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -153,7 +155,7 @@ func (Ovsctl) AddIPSnatRule(bridgeName string, ip net.IP, vlanID int, port strin // Drop other packets which doesn't satisfy above condition cmd = fmt.Sprintf("ovs-ofctl add-flow %v priority=%d,ip,in_port=%s,actions=drop", bridgeName, low, port) - _, err = platform.ExecuteCommand(cmd) + _, err = o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Dropping vlantag packet rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -162,11 +164,11 @@ func (Ovsctl) AddIPSnatRule(bridgeName string, ip net.IP, vlanID int, port strin return nil } -func (Ovsctl) AddArpDnatRule(bridgeName string, port string, mac string) error { +func (o Ovsctl) AddArpDnatRule(bridgeName, port, mac string) error { // Add DNAT rule to forward ARP replies to container interfaces. cmd := fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=2,in_port=%s,actions='mod_dl_dst:ff:ff:ff:ff:ff:ff, load:0x%s->NXM_NX_ARP_THA[],normal'`, bridgeName, port, mac) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding DNAT rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -175,7 +177,7 @@ func (Ovsctl) AddArpDnatRule(bridgeName string, port string, mac string) error { return nil } -func (Ovsctl) AddFakeArpReply(bridgeName string, ip net.IP) error { +func (o Ovsctl) AddFakeArpReply(bridgeName string, ip net.IP) error { // If arp fields matches, set arp reply rule for the request macAddrHex := strings.Replace(defaultMacForArpResponse, ":", "", -1) ipAddrInt := common.IpToInt(ip) @@ -186,7 +188,7 @@ func (Ovsctl) AddFakeArpReply(bridgeName string, ip net.IP) error { move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_TPA[]->NXM_OF_ARP_SPA[], load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_TPA[],IN_PORT'`, bridgeName, high, defaultMacForArpResponse, macAddrHex, ipAddrInt) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -195,14 +197,14 @@ func (Ovsctl) AddFakeArpReply(bridgeName string, ip net.IP) error { return nil } -func (Ovsctl) AddArpReplyRule(bridgeName string, port string, ip net.IP, mac string, vlanid int, mode string) error { +func (o Ovsctl) AddArpReplyRule(bridgeName, port string, ip net.IP, mac string, vlanid int, mode string) error { ipAddrInt := common.IpToInt(ip) macAddrHex := strings.Replace(mac, ":", "", -1) log.Printf("[ovs] Adding ARP reply rule to add vlan %v and forward packet to table 1 for port %v", vlanid, port) cmd := fmt.Sprintf(`ovs-ofctl add-flow %s arp,arp_op=1,in_port=%s,actions='mod_vlan_vid:%v,resubmit(,1)'`, bridgeName, port, vlanid) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -215,7 +217,7 @@ func (Ovsctl) AddArpReplyRule(bridgeName string, port string, ip net.IP, mac str move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[], load:0x%s->NXM_NX_ARP_SHA[],load:0x%x->NXM_OF_ARP_SPA[],strip_vlan,IN_PORT'`, bridgeName, ip.String(), vlanid, high, mac, macAddrHex, ipAddrInt) - _, err = platform.ExecuteCommand(cmd) + _, err = o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding ARP reply rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -225,7 +227,7 @@ func (Ovsctl) AddArpReplyRule(bridgeName string, port string, ip net.IP, mac str } // Add MAC DNAT rule based on dst ip and vlanid -func (Ovsctl) AddMacDnatRule(bridgeName string, port string, ip net.IP, mac string, vlanid int, containerPort string) error { +func (o Ovsctl) AddMacDnatRule(bridgeName, port string, ip net.IP, mac string, vlanid int, containerPort string) error { var cmd string // This rule changes the destination mac to speciifed mac based on the ip and vlanid. // and forwards the packet to corresponding container hostveth port @@ -236,7 +238,7 @@ func (Ovsctl) AddMacDnatRule(bridgeName string, port string, ip net.IP, mac stri } else { cmd = fmt.Sprintf("%s,actions=mod_dl_dst:%s,strip_vlan,%s", commonPrefix, mac, containerPort) } - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding MAC DNAT rule failed with error %v", err) return newErrorOvsctl(err.Error()) @@ -245,32 +247,32 @@ func (Ovsctl) AddMacDnatRule(bridgeName string, port string, ip net.IP, mac stri return nil } -func (Ovsctl) DeleteArpReplyRule(bridgeName string, port string, ip net.IP, vlanid int) { +func (o Ovsctl) DeleteArpReplyRule(bridgeName, port string, ip net.IP, vlanid int) { cmd := fmt.Sprintf("ovs-ofctl del-flows %s arp,arp_op=1,in_port=%s", bridgeName, port) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Deleting ARP reply rule failed with error %v", err) } cmd = fmt.Sprintf("ovs-ofctl del-flows %s table=1,arp,arp_tpa=%s,dl_vlan=%v,arp_op=1", bridgeName, ip.String(), vlanid) - _, err = platform.ExecuteCommand(cmd) + _, err = o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Deleting ARP reply rule failed with error %v", err) } } -func (Ovsctl) DeleteIPSnatRule(bridgeName string, port string) { +func (o Ovsctl) DeleteIPSnatRule(bridgeName, port string) { cmd := fmt.Sprintf("ovs-ofctl del-flows %v ip,in_port=%s", bridgeName, port) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("Error while deleting ovs rule %v error %v", cmd, err) } } -func (Ovsctl) DeleteMacDnatRule(bridgeName string, port string, ip net.IP, vlanid int) { +func (o Ovsctl) DeleteMacDnatRule(bridgeName, port string, ip net.IP, vlanid int) { var cmd string if vlanid != 0 { @@ -281,16 +283,16 @@ func (Ovsctl) DeleteMacDnatRule(bridgeName string, port string, ip net.IP, vlani bridgeName, ip.String(), port) } - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Deleting MAC DNAT rule failed with error %v", err) } } -func (Ovsctl) DeletePortFromOVS(bridgeName string, interfaceName string) error { +func (o Ovsctl) DeletePortFromOVS(bridgeName, interfaceName string) error { // Disconnect external interface from its bridge. cmd := fmt.Sprintf("ovs-vsctl del-port %s %s", bridgeName, interfaceName) - _, err := platform.ExecuteCommand(cmd) + _, err := o.execcli.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Failed to disconnect interface %v from bridge, err:%v.", interfaceName, err) return newErrorOvsctl(err.Error()) diff --git a/platform/mockexec.go b/platform/mockexec.go new file mode 100644 index 0000000000..f768a5c83c --- /dev/null +++ b/platform/mockexec.go @@ -0,0 +1,24 @@ +package platform + +import "errors" + +type mockExecClient struct { + returnError bool +} + +// ErrMockExec - mock exec error +var ErrMockExec = errors.New("mock exec error") + +func NewMockExecClient(returnErr bool) ExecClient { + return &mockExecClient{ + returnError: returnErr, + } +} + +func (e *mockExecClient) ExecuteCommand(string) (string, error) { + if e.returnError { + return "", ErrMockExec + } + + return "", nil +} diff --git a/platform/osInterface.go b/platform/osInterface.go new file mode 100644 index 0000000000..c0f47ed145 --- /dev/null +++ b/platform/osInterface.go @@ -0,0 +1,12 @@ +package platform + +type execClient struct{} + +//nolint:revive // ExecClient make sense +type ExecClient interface { + ExecuteCommand(command string) (string, error) +} + +func NewExecClient() ExecClient { + return &execClient{} +} diff --git a/platform/os_linux.go b/platform/os_linux.go index 0287d8f6a5..96aa0f208c 100644 --- a/platform/os_linux.go +++ b/platform/os_linux.go @@ -47,8 +47,9 @@ func GetOSInfo() string { } func GetProcessSupport() error { + p := &execClient{} cmd := fmt.Sprintf("ps -p %v -o comm=", os.Getpid()) - _, err := ExecuteCommand(cmd) + _, err := p.ExecuteCommand(cmd) return err } @@ -72,7 +73,7 @@ func GetLastRebootTime() (time.Time, error) { return rebootTime.UTC(), nil } -func ExecuteCommand(command string) (string, error) { +func (p *execClient) ExecuteCommand(command string) (string, error) { log.Printf("[Azure-Utils] %s", command) var stderr bytes.Buffer @@ -90,9 +91,10 @@ func ExecuteCommand(command string) (string, error) { } func SetOutboundSNAT(subnet string) error { + p := execClient{} cmd := fmt.Sprintf("iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d %v -j MASQUERADE", subnet) - _, err := ExecuteCommand(cmd) + _, err := p.ExecuteCommand(cmd) if err != nil { log.Printf("SNAT Iptable rule was not set") return err @@ -107,8 +109,9 @@ func ClearNetworkConfiguration() (bool, error) { } func KillProcessByName(processName string) error { + p := &execClient{} cmd := fmt.Sprintf("pkill -f %v", processName) - _, err := ExecuteCommand(cmd) + _, err := p.ExecuteCommand(cmd) return err } @@ -137,9 +140,10 @@ func GetOSDetails() (map[string]string, error) { } func GetProcessNameByID(pidstr string) (string, error) { + p := &execClient{} pidstr = strings.Trim(pidstr, "\n") cmd := fmt.Sprintf("ps -p %s -o comm=", pidstr) - out, err := ExecuteCommand(cmd) + out, err := p.ExecuteCommand(cmd) if err != nil { log.Printf("GetProcessNameByID returned error: %v", err) return "", err @@ -152,10 +156,11 @@ func GetProcessNameByID(pidstr string) (string, error) { } func PrintDependencyPackageDetails() { - out, err := ExecuteCommand("iptables --version") + p := &execClient{} + out, err := p.ExecuteCommand("iptables --version") out = strings.TrimSuffix(out, "\n") log.Printf("[cni-net] iptable version:%s, err:%v", out, err) - out, err = ExecuteCommand("ebtables --version") + out, err = p.ExecuteCommand("ebtables --version") out = strings.TrimSuffix(out, "\n") log.Printf("[cni-net] ebtable version %s, err:%v", out, err) } diff --git a/platform/os_windows.go b/platform/os_windows.go index a5c9ae994d..811bc7fd2b 100644 --- a/platform/os_windows.go +++ b/platform/os_windows.go @@ -87,7 +87,7 @@ func GetLastRebootTime() (time.Time, error) { return rebootTime.UTC(), nil } -func ExecuteCommand(command string) (string, error) { +func (p *execClient) ExecuteCommand(command string) (string, error) { log.Printf("[Azure-Utils] %s", command) var stderr bytes.Buffer @@ -124,8 +124,9 @@ func ClearNetworkConfiguration() (bool, error) { } func KillProcessByName(processName string) { + p := NewExecClient() cmd := fmt.Sprintf("taskkill /IM %v /F", processName) - ExecuteCommand(cmd) + p.ExecuteCommand(cmd) } // ExecutePowershellCommand executes powershell command diff --git a/telemetry/telemetry_windows.go b/telemetry/telemetry_windows.go index 974d4bf717..834ad6b8e2 100644 --- a/telemetry/telemetry_windows.go +++ b/telemetry/telemetry_windows.go @@ -4,10 +4,9 @@ package telemetry import ( + "github.com/Azure/azure-container-networking/platform" "runtime" "strings" - - "github.com/Azure/azure-container-networking/platform" ) const ( @@ -38,8 +37,9 @@ func (report *CNIReport) GetSystemDetails() { } func (report *CNIReport) GetOSDetails() { + p := platform.NewExecClient() report.OSDetails = OSInfo{OSType: runtime.GOOS} - out, err := platform.ExecuteCommand(versionCmd) + out, err := p.ExecuteCommand(versionCmd) if err == nil { report.OSDetails.OSVersion = strings.Replace(out, delimiter, "", -1) }