From 8f64c08ca4b58e6f710304ccdd411de2915f4099 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 5 Jul 2022 17:01:47 -0700 Subject: [PATCH 01/86] Native Endpoint Client Add Endpoints --- go.mod | 3 +- go.sum | 4 + network/endpoint_linux.go | 48 ++++-- network/native_endpointclient_linux.go | 220 +++++++++++++++++++++++++ network/network.go | 1 + network/network_linux.go | 3 + 6 files changed, 264 insertions(+), 15 deletions(-) create mode 100644 network/native_endpointclient_linux.go diff --git a/go.mod b/go.mod index b716bbe468..8acdb5db3e 100644 --- a/go.mod +++ b/go.mod @@ -104,7 +104,8 @@ require ( github.com/subosito/gotenv v1.3.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect - github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect + github.com/vishvananda/netlink v1.2.1-beta.2 + github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.6.0 // indirect diff --git a/go.sum b/go.sum index 184c5ebaca..1719c6b73e 100644 --- a/go.sum +++ b/go.sum @@ -832,11 +832,15 @@ github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= +github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg= +github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index b084895097..f983f502ea 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -58,6 +58,8 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p var localIP string var epClient EndpointClient var vlanid int = 0 + var vnetIfName string + var vnetNSName string if nw.Endpoints[epInfo.Id] != nil { log.Printf("[net] Endpoint alreday exists.") @@ -89,21 +91,39 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p } if vlanid != 0 { - log.Printf("OVS client") - if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { - nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) - } + if nw.Mode == opModeNative { + log.Printf("Mode %s", nw.Mode) + log.Printf("Native client") + vnetIfName = fmt.Sprintf("eth0.%s", vlanid) + vnetNSName = fmt.Sprintf("az_ns_%s", vlanid) + + epClient = NewNativeEndpointClient( + nw.extIf, + hostIfName, + vnetIfName, + contIfName, + vnetNSName, + nw.Mode, + vlanid, + nl, + plc) + } else { + log.Printf("OVS client") + if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { + nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) + } - epClient = NewOVSEndpointClient( - nw, - epInfo, - hostIfName, - contIfName, - vlanid, - localIP, - nl, - ovsctl.NewOvsctl(), - plc) + epClient = NewOVSEndpointClient( + nw, + epInfo, + hostIfName, + contIfName, + vlanid, + localIP, + nl, + ovsctl.NewOvsctl(), + plc) + } } else if nw.Mode != opModeTransparent { log.Printf("Bridge client") epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go new file mode 100644 index 0000000000..cb6f4c88b4 --- /dev/null +++ b/network/native_endpointclient_linux.go @@ -0,0 +1,220 @@ +package network + +import ( + "errors" + "fmt" + "net" + + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/platform" + + vishnetlink "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" +) + +var errorNativeEndpointClient = errors.New("TransparentEndpointClient Error") + +func newErrorNativeEndpointClient(errStr string) error { + return fmt.Errorf("%w : %s", errorTransparentEndpointClient, errStr) +} + +type NativeEndpointClient struct { + hostPrimaryIfName string //So like eth0 + hostVethName string //So like eth0.X + vnetVethName string //Peer is containerVethName + containerVethName string //Peer is vnetVethName + + hostPrimaryMac net.HardwareAddr + vnetMac net.HardwareAddr + containerMac net.HardwareAddr + hostVethMac net.HardwareAddr + + vnetNSName string + vnetNS *Namespace + dns *DNSInfo + + mode string + vlanID int + netlink netlink.NetlinkInterface + netioshim netio.NetIOInterface + plClient platform.ExecClient + netUtilsClient networkutils.NetworkUtils +} + +func NewNativeEndpointClient( + extIf *externalInterface, + hostVethName string, + vnetVethName string, + containerVethName string, + vnetNSName string, + mode string, + vlanid int, + nl netlink.NetlinkInterface, + plc platform.ExecClient, +) *NativeEndpointClient { + + client := &NativeEndpointClient{ + hostPrimaryIfName: extIf.Name, + hostVethName: hostVethName, + vnetVethName: vnetVethName, + containerVethName: containerVethName, + hostPrimaryMac: extIf.MacAddress, + vnetNSName: vnetNSName, + mode: mode, + vlanID: vlanid, + netlink: nl, + netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + } + + return client +} + +func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { + var err error + //Create the vnet namespace + if _, err = netns.NewNamed(client.vnetNSName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + //Create said namespace (doesn't enter or exit yet) + vnetNS, err := OpenNamespace(client.vnetNSName) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + client.vnetNS = vnetNS + + //Create the host vlan link and move it to vnet NS + linkAttrs := vishnetlink.NewLinkAttrs() + linkAttrs.Name = client.hostVethName + //TODO: linkAttrs.ParentIndex = eth0 (Primary) + link := &vishnetlink.Vlan{ + LinkAttrs: linkAttrs, + VlanId: client.vlanID, + } + vishnetlink.LinkAdd(link) + if err = client.netlink.SetLinkNetNs(client.hostVethName, client.vnetNS.GetFd()); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + //Create veth pair (automatically set to UP) + if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + //Move vnetVethName into vnet namespace (peer will be moved in MoveEndpointsToContainerNS) + if err = client.netlink.SetLinkNetNs(client.vnetVethName, client.vnetNS.GetFd()); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + //If there is a failure, delete the links + defer func() { + if err != nil { + //Delete vnet <> container + if delErr := client.netlink.DeleteLink(client.hostVethName); delErr != nil { + log.Errorf("Deleting veth failed on addendpoint failure:%v", delErr) + } + //Delete eth0 <> eth0.X + + } + }() + //Check that container veth exists. DOES it matter if they are in a different NS? + containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + client.containerMac = containerIf.HardwareAddr + + //Check that host veth exists (eth0.X) + hostVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + client.hostVethMac = hostVethIf.HardwareAddr + + //Check that vnet veth exists + vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + client.vnetMac = vnetVethIf.HardwareAddr + //Set MTU? + + return nil +} +func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { + //Described as rules on ip addresses on the container interface, + + // ip route add dev + // This route is needed for incoming packets to pod to route via hostveth + //Transparent has routes going to host, but we don't need that here, right? + //Each endpoint only has two routes, which are the default. + + //What is arp proxy? Set up arp rules here? + //AddEndpointRules vs. ConfigureContainerInterfacesAndRoutes + return nil +} +func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { + return nil +} +func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { + if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + return nil +} +func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { + if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + client.containerVethName = epInfo.IfName + + return nil +} +func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + // add route for virtualgwip + // ip route add 169.254.1.1/32 dev eth0, but where do you specify eth0? + // addRoutes says it adds route to containerVethName, so this appears to be a route to the container? + + virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) + routeInfo := RouteInfo{ + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + } + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + return newErrorTransparentEndpointClient(err.Error()) + } + + // ip route add default via 169.254.1.1 dev eth0 + _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) + dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} + routeInfo = RouteInfo{ + Dst: dstIP, + Gw: virtualGwIP, + } + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + return err + } + + // arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac + log.Printf("[net] Adding static arp for IP address %v and MAC %v in Container namespace", + virtualGwNet.String(), client.hostVethMac) + if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, + client.containerVethName, + virtualGwNet.IP, + client.hostVethMac, + false); err != nil { + return fmt.Errorf("Adding arp in container failed: %w", err) + } + + return nil +} +func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { + return nil +} diff --git a/network/network.go b/network/network.go index 16979d7414..a28d0a7244 100644 --- a/network/network.go +++ b/network/network.go @@ -18,6 +18,7 @@ const ( opModeBridge = "bridge" opModeTunnel = "tunnel" opModeTransparent = "transparent" + opModeNative = "native" opModeDefault = opModeTunnel ) diff --git a/network/network_linux.go b/network/network_linux.go index 2afb4a5a0f..5ced57c8c2 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -88,6 +88,9 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt return nil, fmt.Errorf("Ipv6 forwarding failed: %w", err) } } + case opModeNative: + log.Printf("Native mode") + ifName = extIf.Name default: return nil, errNetworkModeInvalid } From 2b3cb7aa75d3bf983805f4613c8db3ed501dedd5 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 6 Jul 2022 10:44:42 -0700 Subject: [PATCH 02/86] AddEndpointRules, ConfigureContainerInterfacesAndRoutes --- network/endpoint_linux.go | 4 +- network/native_endpointclient_linux.go | 167 +++++++++++++++++++------ 2 files changed, 130 insertions(+), 41 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index f983f502ea..4d8eb08a0e 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -94,8 +94,8 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p if nw.Mode == opModeNative { log.Printf("Mode %s", nw.Mode) log.Printf("Native client") - vnetIfName = fmt.Sprintf("eth0.%s", vlanid) - vnetNSName = fmt.Sprintf("az_ns_%s", vlanid) + vnetIfName = fmt.Sprintf("eth0.%d", vlanid) + vnetNSName = fmt.Sprintf("az_ns_%d", vlanid) epClient = NewNativeEndpointClient( nw.extIf, diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index cb6f4c88b4..7cf7da9353 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -15,10 +15,14 @@ import ( "github.com/vishvananda/netns" ) -var errorNativeEndpointClient = errors.New("TransparentEndpointClient Error") +const ( + azureMac = "12:34:56:78:9a:bc" +) + +var errorNativeEndpointClient = errors.New("NativeEndpointClient Error") func newErrorNativeEndpointClient(errStr string) error { - return fmt.Errorf("%w : %s", errorTransparentEndpointClient, errStr) + return fmt.Errorf("%w : %s", errorNativeEndpointClient, errStr) } type NativeEndpointClient struct { @@ -33,7 +37,6 @@ type NativeEndpointClient struct { hostVethMac net.HardwareAddr vnetNSName string - vnetNS *Namespace dns *DNSInfo mode string @@ -85,39 +88,45 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if err != nil { return newErrorNativeEndpointClient(err.Error()) } - client.vnetNS = vnetNS + defer vnetNS.Close() //Create the host vlan link and move it to vnet NS linkAttrs := vishnetlink.NewLinkAttrs() linkAttrs.Name = client.hostVethName - //TODO: linkAttrs.ParentIndex = eth0 (Primary) + //Get parent interface index. Is the index the same regardless of lib used? + eth0, err := client.netioshim.GetNetworkInterfaceByName(client.hostPrimaryIfName) + //Is this how you set the peer? + linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ LinkAttrs: linkAttrs, VlanId: client.vlanID, } vishnetlink.LinkAdd(link) - if err = client.netlink.SetLinkNetNs(client.hostVethName, client.vnetNS.GetFd()); err != nil { + if err = client.netlink.SetLinkNetNs(client.hostVethName, vnetNS.GetFd()); err != nil { return newErrorNativeEndpointClient(err.Error()) } + vishnetlink.LinkSetUp(link) //Create veth pair (automatically set to UP) if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } //Move vnetVethName into vnet namespace (peer will be moved in MoveEndpointsToContainerNS) - if err = client.netlink.SetLinkNetNs(client.vnetVethName, client.vnetNS.GetFd()); err != nil { + if err = client.netlink.SetLinkNetNs(client.vnetVethName, vnetNS.GetFd()); err != nil { return newErrorNativeEndpointClient(err.Error()) } //If there is a failure, delete the links defer func() { if err != nil { - //Delete vnet <> container + //Delete vnet <-> container + if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { + log.Errorf("Deleting vnetVeth failed on addendpoint failure:%v", delErr) + } + //Delete eth0 <-> eth0.X if delErr := client.netlink.DeleteLink(client.hostVethName); delErr != nil { - log.Errorf("Deleting veth failed on addendpoint failure:%v", delErr) + log.Errorf("Deleting hostVeth failed on addendpoint failure:%v", delErr) } - //Delete eth0 <> eth0.X - } }() //Check that container veth exists. DOES it matter if they are in a different NS? @@ -154,41 +163,77 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error //What is arp proxy? Set up arp rules here? //AddEndpointRules vs. ConfigureContainerInterfacesAndRoutes - return nil -} -func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { - return nil -} -func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { - if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { + + //Is it okay if I enter the namespace and then exit and enter again? + //Is it better just to call OpenNamespace every time? + + // Open the vnet network namespace + log.Printf("Opening vnetns %v.", client.vnetNSName) + ns, err := OpenNamespace(client.vnetNSName) + if err != nil { + return err + } + defer ns.Close() + // Enter the vnet network namespace + log.Printf("Entering vnetns %v.", ns) + if err := ns.Enter(); err != nil { + return err + } + + // Exit vnet network namespace + defer func() { + if err := ns.Exit(); err != nil { + log.Printf("Could not exit vnetns, err:%v.", err) + } + }() + + var routeInfoList []RouteInfo + + // ip route add dev + // This route is needed for incoming packets to pod to route via hostveth + for _, ipAddr := range epInfo.IPAddresses { + var ( + routeInfo RouteInfo + ipNet net.IPNet + ) + + if ipAddr.IP.To4() != nil { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} + } else { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} + } + log.Printf("[net] Adding route for the ip %v", ipNet.String()) + routeInfo.Dst = ipNet + routeInfoList = append(routeInfoList, routeInfo) + + } + if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return newErrorNativeEndpointClient(err.Error()) } - return nil -} -func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + // Add default/gateway routes (Assuming indempotent) + if err = client.AddDefaultRoutes(client.hostVethName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + // Add ARP entry (Assuming indempotent) + if err = client.AddDefaultArp(client.vnetVethName, azureMac); err != nil { return newErrorNativeEndpointClient(err.Error()) } - client.containerVethName = epInfo.IfName return nil } -func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } +// Helper that creates routing rules for the current NS which direct packets +// to the virtual gateway ip on linkToName device interface +func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { // add route for virtualgwip - // ip route add 169.254.1.1/32 dev eth0, but where do you specify eth0? - // addRoutes says it adds route to containerVethName, so this appears to be a route to the container? - + // ip route add 169.254.1.1/32 dev eth0 virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ Dst: *virtualGwNet, Scope: netlink.RT_SCOPE_LINK, } - if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { - return newErrorTransparentEndpointClient(err.Error()) + if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { + return err } // ip route add default via 169.254.1.1 dev eth0 @@ -198,19 +243,63 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo Dst: dstIP, Gw: virtualGwIP, } - if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { return err } + return nil +} - // arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac - log.Printf("[net] Adding static arp for IP address %v and MAC %v in Container namespace", - virtualGwNet.String(), client.hostVethMac) +// Helper that creates arp entry for the current NS which maps the virtual +// gateway to destMac +func (client *NativeEndpointClient) AddDefaultArp(name string, destMac string) error { + _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) + // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac + log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", + virtualGwNet.String(), destMac) + hardwareAddr, err := net.ParseMAC(destMac) + if err != nil { + return err + } if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, - client.containerVethName, + name, //What is the purpose of name? virtualGwNet.IP, - client.hostVethMac, + hardwareAddr, false); err != nil { - return fmt.Errorf("Adding arp in container failed: %w", err) + return fmt.Errorf("adding arp entry failed: %w", err) + } + return nil +} +func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { + +} +func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { + if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + return nil +} +func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { + if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + client.containerVethName = epInfo.IfName + + return nil +} +func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + // add route for virtualgwip + // ip route add 169.254.1.1/32 dev eth0, but where do you specify eth0? + // addRoutes says it adds route to containerVethName, so this appears to be a route to the container? + //Currently called from container NS + if err := client.AddDefaultRoutes(client.containerVethName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + if err := client.AddDefaultArp(epInfo.NetNsPath, client.vnetMac.String()); err != nil { + return newErrorNativeEndpointClient(err.Error()) } return nil From a85c3656cb846f252048ff4a19d50ca3dd596b38 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 6 Jul 2022 12:31:46 -0700 Subject: [PATCH 03/86] Changed interface names, log statements nw.extIf.Name > eth0 (eth0) eth0.vlanid > eth0.X (eth0.1) %s%s hostIfName > vnet (A1veth0) %s%s-2 contIfName > container (B1veth0) --- network/endpoint_linux.go | 10 +-- network/native_endpointclient_linux.go | 103 +++++++++++++++---------- 2 files changed, 67 insertions(+), 46 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 4d8eb08a0e..c59ee0a813 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -58,7 +58,7 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p var localIP string var epClient EndpointClient var vlanid int = 0 - var vnetIfName string + var ethXIfName string var vnetNSName string if nw.Endpoints[epInfo.Id] != nil { @@ -94,13 +94,13 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p if nw.Mode == opModeNative { log.Printf("Mode %s", nw.Mode) log.Printf("Native client") - vnetIfName = fmt.Sprintf("eth0.%d", vlanid) + ethXIfName = fmt.Sprintf("eth0.%d", vlanid) vnetNSName = fmt.Sprintf("az_ns_%d", vlanid) - + //hostIfName may be a misnomer as this end is in the vnet NS epClient = NewNativeEndpointClient( - nw.extIf, + nw.extIf.Name, + ethXIfName, hostIfName, - vnetIfName, contIfName, vnetNSName, nw.Mode, diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 7cf7da9353..8b4773293c 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net" + "os" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" @@ -26,18 +27,17 @@ func newErrorNativeEndpointClient(errStr string) error { } type NativeEndpointClient struct { - hostPrimaryIfName string //So like eth0 - hostVethName string //So like eth0.X + eth0VethName string //So like eth0 + ethXVethName string //So like eth0.X vnetVethName string //Peer is containerVethName containerVethName string //Peer is vnetVethName - hostPrimaryMac net.HardwareAddr - vnetMac net.HardwareAddr - containerMac net.HardwareAddr - hostVethMac net.HardwareAddr + //hostPrimaryMac net.HardwareAddr + vnetMac net.HardwareAddr + containerMac net.HardwareAddr + hostVethMac net.HardwareAddr vnetNSName string - dns *DNSInfo mode string vlanID int @@ -48,8 +48,8 @@ type NativeEndpointClient struct { } func NewNativeEndpointClient( - extIf *externalInterface, - hostVethName string, + eth0VethName string, + ethXVethName string, vnetVethName string, containerVethName string, vnetNSName string, @@ -58,20 +58,21 @@ func NewNativeEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, ) *NativeEndpointClient { - + log.Printf("Create new native client: eth0:%s,ethX:%s,vnet:%s,cont:%s,id:%s", + eth0VethName, ethXVethName, vnetVethName, containerVethName, vlanid) client := &NativeEndpointClient{ - hostPrimaryIfName: extIf.Name, - hostVethName: hostVethName, + eth0VethName: eth0VethName, + ethXVethName: ethXVethName, vnetVethName: vnetVethName, containerVethName: containerVethName, - hostPrimaryMac: extIf.MacAddress, - vnetNSName: vnetNSName, - mode: mode, - vlanID: vlanid, - netlink: nl, - netioshim: &netio.NetIO{}, - plClient: plc, - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + //hostPrimaryMac: extIf.MacAddress, + vnetNSName: vnetNSName, + mode: mode, + vlanID: vlanid, + netlink: nl, + netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), } return client @@ -79,22 +80,22 @@ func NewNativeEndpointClient( func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { var err error - //Create the vnet namespace + log.Printf("Create the vnet namespace") if _, err = netns.NewNamed(client.vnetNSName); err != nil { return newErrorNativeEndpointClient(err.Error()) } - //Create said namespace (doesn't enter or exit yet) + log.Printf("Open said namespace and get NS reference(doesn't enter or exit yet)") vnetNS, err := OpenNamespace(client.vnetNSName) if err != nil { return newErrorNativeEndpointClient(err.Error()) } defer vnetNS.Close() - //Create the host vlan link and move it to vnet NS + log.Printf("Create the host vlan link") linkAttrs := vishnetlink.NewLinkAttrs() - linkAttrs.Name = client.hostVethName + linkAttrs.Name = client.ethXVethName //Get parent interface index. Is the index the same regardless of lib used? - eth0, err := client.netioshim.GetNetworkInterfaceByName(client.hostPrimaryIfName) + eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) //Is this how you set the peer? linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ @@ -102,16 +103,17 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { VlanId: client.vlanID, } vishnetlink.LinkAdd(link) - if err = client.netlink.SetLinkNetNs(client.hostVethName, vnetNS.GetFd()); err != nil { + log.Printf("Move vlan link to vnet NS") + if err = client.netlink.SetLinkNetNs(client.ethXVethName, vnetNS.GetFd()); err != nil { return newErrorNativeEndpointClient(err.Error()) } vishnetlink.LinkSetUp(link) - //Create veth pair (automatically set to UP) + log.Printf("Create veth pair (automatically set to UP)") if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } - //Move vnetVethName into vnet namespace (peer will be moved in MoveEndpointsToContainerNS) + log.Printf("Move vnetVethName into vnet namespace (peer will be moved in MoveEndpointsToContainerNS)") if err = client.netlink.SetLinkNetNs(client.vnetVethName, vnetNS.GetFd()); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -119,32 +121,33 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { //If there is a failure, delete the links defer func() { if err != nil { + log.Printf("Failure detected, deleting links...") //Delete vnet <-> container if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { log.Errorf("Deleting vnetVeth failed on addendpoint failure:%v", delErr) } //Delete eth0 <-> eth0.X - if delErr := client.netlink.DeleteLink(client.hostVethName); delErr != nil { + if delErr := client.netlink.DeleteLink(client.ethXVethName); delErr != nil { log.Errorf("Deleting hostVeth failed on addendpoint failure:%v", delErr) } } }() - //Check that container veth exists. DOES it matter if they are in a different NS? + log.Printf("Check that container veth exists.") containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) if err != nil { return newErrorNativeEndpointClient(err.Error()) } client.containerMac = containerIf.HardwareAddr - //Check that host veth exists (eth0.X) - hostVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName) + log.Printf("Check that (eth0.X) exists") + hostVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) if err != nil { return newErrorNativeEndpointClient(err.Error()) } client.hostVethMac = hostVethIf.HardwareAddr - //Check that vnet veth exists - vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName) + log.Printf("Check that vnet veth exists") + vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) if err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -182,6 +185,7 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error // Exit vnet network namespace defer func() { + log.Printf("Exiting vnetns %v.", ns) if err := ns.Exit(); err != nil { log.Printf("Could not exit vnetns, err:%v.", err) } @@ -202,7 +206,7 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error } else { ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} } - log.Printf("[net] Adding route for the ip %v", ipNet.String()) + log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) routeInfo.Dst = ipNet routeInfoList = append(routeInfoList, routeInfo) @@ -210,11 +214,11 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return newErrorNativeEndpointClient(err.Error()) } - // Add default/gateway routes (Assuming indempotent) - if err = client.AddDefaultRoutes(client.hostVethName); err != nil { + log.Printf("Vnet NS add default/gateway routes (Assuming indempotent)") + if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } - // Add ARP entry (Assuming indempotent) + log.Printf("Vnet NS add default ARP entry (Assuming indempotent)") if err = client.AddDefaultArp(client.vnetVethName, azureMac); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -225,7 +229,7 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error // Helper that creates routing rules for the current NS which direct packets // to the virtual gateway ip on linkToName device interface func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { - // add route for virtualgwip + log.Printf("Add route for virtualgwip") // ip route add 169.254.1.1/32 dev eth0 virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ @@ -236,7 +240,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { return err } - // ip route add default via 169.254.1.1 dev eth0 + log.Printf("Add default route (ip route add default via 169.254.1.1 dev eth0)") _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} routeInfo = RouteInfo{ @@ -273,12 +277,14 @@ func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { + log.Printf("Moving endpoint to container NS") if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { return newErrorNativeEndpointClient(err.Error()) } return nil } func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { + log.Printf("Setup container interface") if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -287,21 +293,36 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf return nil } func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + log.Printf("Assign IPs to container veth interface") if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorNativeEndpointClient(err.Error()) } - // add route for virtualgwip + log.Printf("Container NS add route for virtualgwip") // ip route add 169.254.1.1/32 dev eth0, but where do you specify eth0? // addRoutes says it adds route to containerVethName, so this appears to be a route to the container? //Currently called from container NS if err := client.AddDefaultRoutes(client.containerVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } + log.Printf("Container NS add arp entry") if err := client.AddDefaultArp(epInfo.NetNsPath, client.vnetMac.String()); err != nil { return newErrorNativeEndpointClient(err.Error()) } + log.Printf("Create resolv.conf for DNS") + folder := fmt.Sprintf("/etc/netns/%s", client.vnetNSName) + resolv := fmt.Sprintf("%s/resolv.conf", folder) + if _, err := os.Stat(folder); os.IsNotExist(err) { + os.MkdirAll(folder, 0700) // Create your file + } + log.Printf("Writing to resolv.conf file %s , %s", epInfo.DNS.Servers, epInfo.DNS.Servers[0]) + data := []byte(fmt.Sprintf("nameserver %s", epInfo.DNS.Servers[0])) + err := os.WriteFile(resolv, data, 0644) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + return nil } func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { From f08f0b73e3ebc58e7dfa29a6ec4fbef08f94d343 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 7 Jul 2022 16:44:33 -0700 Subject: [PATCH 04/86] Renaming, using lib to set ns --- network/native_endpointclient_linux.go | 218 +++++++++++++++---------- 1 file changed, 128 insertions(+), 90 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 8b4773293c..440d901034 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "net" - "os" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" @@ -35,9 +34,12 @@ type NativeEndpointClient struct { //hostPrimaryMac net.HardwareAddr vnetMac net.HardwareAddr containerMac net.HardwareAddr - hostVethMac net.HardwareAddr + ethXMac net.HardwareAddr + + vmNS netns.NsHandle vnetNSName string + vnetNS netns.NsHandle mode string vlanID int @@ -80,31 +82,42 @@ func NewNativeEndpointClient( func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { var err error - log.Printf("Create the vnet namespace") - if _, err = netns.NewNamed(client.vnetNSName); err != nil { + log.Printf("Get VM namespace handle") + vmNS, err := netns.Get() + if err != nil { return newErrorNativeEndpointClient(err.Error()) } - log.Printf("Open said namespace and get NS reference(doesn't enter or exit yet)") - vnetNS, err := OpenNamespace(client.vnetNSName) - if err != nil { + log.Printf("Save VM namespace: %s", vmNS) + client.vmNS = vmNS + + log.Printf("Create the vnet namespace") + if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { return newErrorNativeEndpointClient(err.Error()) } - defer vnetNS.Close() + log.Printf("Set current namespace to VM: %s", vmNS) + netns.Set(vmNS) - log.Printf("Create the host vlan link") + log.Printf("Create the host vlan link after getting %s", client.eth0VethName) linkAttrs := vishnetlink.NewLinkAttrs() linkAttrs.Name = client.ethXVethName //Get parent interface index. Is the index the same regardless of lib used? eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) + if err != nil { + log.Printf("Failed to get interface: %s", client.eth0VethName) + return newErrorNativeEndpointClient(err.Error()) + } //Is this how you set the peer? linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ LinkAttrs: linkAttrs, VlanId: client.vlanID, } - vishnetlink.LinkAdd(link) - log.Printf("Move vlan link to vnet NS") - if err = client.netlink.SetLinkNetNs(client.ethXVethName, vnetNS.GetFd()); err != nil { + err = vishnetlink.LinkAdd(link) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + log.Printf("Move vlan link to vnet NS: %d", uintptr(client.vnetNS)) + if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNS)); err != nil { return newErrorNativeEndpointClient(err.Error()) } vishnetlink.LinkSetUp(link) @@ -114,13 +127,15 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return newErrorNativeEndpointClient(err.Error()) } log.Printf("Move vnetVethName into vnet namespace (peer will be moved in MoveEndpointsToContainerNS)") - if err = client.netlink.SetLinkNetNs(client.vnetVethName, vnetNS.GetFd()); err != nil { + if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNS)); err != nil { return newErrorNativeEndpointClient(err.Error()) } //If there is a failure, delete the links defer func() { if err != nil { + log.Printf("Switching NS to vnet") + netns.Set(client.vnetNS) log.Printf("Failure detected, deleting links...") //Delete vnet <-> container if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { @@ -130,6 +145,8 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if delErr := client.netlink.DeleteLink(client.ethXVethName); delErr != nil { log.Errorf("Deleting hostVeth failed on addendpoint failure:%v", delErr) } + log.Printf("Switching NS to vm") + netns.Set(client.vmNS) } }() log.Printf("Check that container veth exists.") @@ -139,12 +156,21 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } client.containerMac = containerIf.HardwareAddr + log.Printf("Switch NS to vnet") + netns.Set(client.vnetNS) + + currNS, err := netns.Get() + log.Printf("Current NS after switch to vnet: %v.", currNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + log.Printf("Check that (eth0.X) exists") - hostVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) + ethXVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) if err != nil { return newErrorNativeEndpointClient(err.Error()) } - client.hostVethMac = hostVethIf.HardwareAddr + client.ethXMac = ethXVethIf.HardwareAddr log.Printf("Check that vnet veth exists") vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) @@ -153,75 +179,14 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } client.vnetMac = vnetVethIf.HardwareAddr //Set MTU? + log.Printf("Switch NS to vm") + netns.Set(client.vmNS) return nil } func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { - //Described as rules on ip addresses on the container interface, - - // ip route add dev - // This route is needed for incoming packets to pod to route via hostveth - //Transparent has routes going to host, but we don't need that here, right? - //Each endpoint only has two routes, which are the default. - - //What is arp proxy? Set up arp rules here? - //AddEndpointRules vs. ConfigureContainerInterfacesAndRoutes - - //Is it okay if I enter the namespace and then exit and enter again? - //Is it better just to call OpenNamespace every time? - - // Open the vnet network namespace - log.Printf("Opening vnetns %v.", client.vnetNSName) - ns, err := OpenNamespace(client.vnetNSName) - if err != nil { - return err - } - defer ns.Close() - // Enter the vnet network namespace - log.Printf("Entering vnetns %v.", ns) - if err := ns.Enter(); err != nil { - return err - } - - // Exit vnet network namespace - defer func() { - log.Printf("Exiting vnetns %v.", ns) - if err := ns.Exit(); err != nil { - log.Printf("Could not exit vnetns, err:%v.", err) - } - }() - - var routeInfoList []RouteInfo - - // ip route add dev - // This route is needed for incoming packets to pod to route via hostveth - for _, ipAddr := range epInfo.IPAddresses { - var ( - routeInfo RouteInfo - ipNet net.IPNet - ) - - if ipAddr.IP.To4() != nil { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} - } else { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} - } - log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) - routeInfo.Dst = ipNet - routeInfoList = append(routeInfoList, routeInfo) - - } - if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - log.Printf("Vnet NS add default/gateway routes (Assuming indempotent)") - if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - log.Printf("Vnet NS add default ARP entry (Assuming indempotent)") - if err = client.AddDefaultArp(client.vnetVethName, azureMac); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } + //There are no rules to add here + //Described as rules on ip addresses on the container interface return nil } @@ -233,8 +198,9 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { // ip route add 169.254.1.1/32 dev eth0 virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ - Dst: *virtualGwNet, - Scope: netlink.RT_SCOPE_LINK, + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + DevName: linkToName, } if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { return err @@ -244,9 +210,11 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} routeInfo = RouteInfo{ - Dst: dstIP, - Gw: virtualGwIP, + Dst: dstIP, + Gw: virtualGwIP, + DevName: linkToName, } + if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { return err } @@ -255,7 +223,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { // Helper that creates arp entry for the current NS which maps the virtual // gateway to destMac -func (client *NativeEndpointClient) AddDefaultArp(name string, destMac string) error { +func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", @@ -265,7 +233,7 @@ func (client *NativeEndpointClient) AddDefaultArp(name string, destMac string) e return err } if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, - name, //What is the purpose of name? + interfaceName, //What is the purpose of name? virtualGwNet.IP, hardwareAddr, false); err != nil { @@ -293,8 +261,17 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf return nil } func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + log.Printf("Setting NS to container path %d", uintptr(client.vnetNS)) + contNS, err := netns.GetFromPath(epInfo.NetNsPath) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + err = netns.Set(contNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } log.Printf("Assign IPs to container veth interface") - if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + if err = client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -306,10 +283,10 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } log.Printf("Container NS add arp entry") - if err := client.AddDefaultArp(epInfo.NetNsPath, client.vnetMac.String()); err != nil { + if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { return newErrorNativeEndpointClient(err.Error()) } - + /* Ignore the resolv.conf for now log.Printf("Create resolv.conf for DNS") folder := fmt.Sprintf("/etc/netns/%s", client.vnetNSName) resolv := fmt.Sprintf("%s/resolv.conf", folder) @@ -318,7 +295,68 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo } log.Printf("Writing to resolv.conf file %s , %s", epInfo.DNS.Servers, epInfo.DNS.Servers[0]) data := []byte(fmt.Sprintf("nameserver %s", epInfo.DNS.Servers[0])) - err := os.WriteFile(resolv, data, 0644) + if err := os.WriteFile(resolv, data, 0644); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + */ + // ip route add dev + //Each endpoint only has two routes, which are the default. + + //What is arp proxy? Set up arp rules here? + //AddEndpointRules vs. ConfigureContainerInterfacesAndRoutes + currNS, err := netns.Get() + log.Printf("Current NS before switch: %v.", currNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + log.Printf("Opening vnetns %v.", client.vnetNSName) + log.Printf("Setting NS to vnet %d", uintptr(client.vnetNS)) + netns.Set(client.vnetNS) //Catch error later + currNS, err = netns.Get() + + log.Printf("Current NS after switch: %v.", currNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + var routeInfoList []RouteInfo + + // ip route add dev + // This route is needed for incoming packets to pod to route via hostveth + for _, ipAddr := range epInfo.IPAddresses { + var ( + routeInfo RouteInfo + ipNet net.IPNet + ) + + if ipAddr.IP.To4() != nil { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} + } else { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} + } + log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) + routeInfo.Dst = ipNet + routeInfoList = append(routeInfoList, routeInfo) + + } + log.Printf("Client data: ethX: %s, vnet: %s", client.ethXVethName, client.vnetVethName) + + log.Printf("Vnet NS add default/gateway routes (Assuming indempotent)") + if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + log.Printf("Vnet NS add default ARP entry (Assuming indempotent)") + if err = client.AddDefaultArp(client.ethXVethName, azureMac); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + log.Printf("Adding routes") + if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + log.Printf("Return to container NS") + err = netns.Set(contNS) if err != nil { return newErrorNativeEndpointClient(err.Error()) } From 8531563fc5993e8ed9afcbdac9036c0b1b7fa422 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 7 Jul 2022 17:04:43 -0700 Subject: [PATCH 05/86] Namespace "path" is /var/run/netns/ --- network/native_endpointclient_linux.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 440d901034..1c451986e2 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -310,10 +310,26 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } - log.Printf("Opening vnetns %v.", client.vnetNSName) - log.Printf("Setting NS to vnet %d", uintptr(client.vnetNS)) - netns.Set(client.vnetNS) //Catch error later - currNS, err = netns.Get() + // Open the vnet network namespace + log.Printf("Opening vnetns %v.", fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) + ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) + if err != nil { + return err + } + defer ns.Close() + // Enter the vnet network namespace + log.Printf("Entering vnetns %v.", ns) + if err := ns.Enter(); err != nil { + return err + } + + // Exit vnet network namespace + defer func() { + log.Printf("Exiting vnetns %v.", ns) + if err := ns.Exit(); err != nil { + log.Printf("Could not exit vnetns, err:%v.", err) + } + }() log.Printf("Current NS after switch: %v.", currNS) if err != nil { From 3752a41bc4fb6688037c4bc1d19906c978f95e92 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 8 Jul 2022 14:39:55 -0700 Subject: [PATCH 06/86] Loopback set up, Remove auto kernel subnet route --- network/native_endpointclient_linux.go | 83 +++++++++++++++----------- 1 file changed, 49 insertions(+), 34 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 1c451986e2..1f6a666786 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -16,7 +16,8 @@ import ( ) const ( - azureMac = "12:34:56:78:9a:bc" + azureMac = "12:34:56:78:9a:bc" + loopbackIf = "lo" ) var errorNativeEndpointClient = errors.New("NativeEndpointClient Error") @@ -31,7 +32,6 @@ type NativeEndpointClient struct { vnetVethName string //Peer is containerVethName containerVethName string //Peer is vnetVethName - //hostPrimaryMac net.HardwareAddr vnetMac net.HardwareAddr containerMac net.HardwareAddr ethXMac net.HardwareAddr @@ -60,21 +60,20 @@ func NewNativeEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, ) *NativeEndpointClient { - log.Printf("Create new native client: eth0:%s,ethX:%s,vnet:%s,cont:%s,id:%s", + log.Printf("Create new native client: eth0:%s, ethX:%s, vnet:%s, cont:%s, id:%s", eth0VethName, ethXVethName, vnetVethName, containerVethName, vlanid) client := &NativeEndpointClient{ eth0VethName: eth0VethName, ethXVethName: ethXVethName, vnetVethName: vnetVethName, containerVethName: containerVethName, - //hostPrimaryMac: extIf.MacAddress, - vnetNSName: vnetNSName, - mode: mode, - vlanID: vlanid, - netlink: nl, - netioshim: &netio.NetIO{}, - plClient: plc, - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + vnetNSName: vnetNSName, + mode: mode, + vlanID: vlanid, + netlink: nl, + netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), } return client @@ -90,43 +89,48 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { log.Printf("Save VM namespace: %s", vmNS) client.vmNS = vmNS - log.Printf("Create the vnet namespace") + log.Printf("Create the vnet namespace and switch to it") if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { return newErrorNativeEndpointClient(err.Error()) } + log.Printf("Set current namespace to VM: %s", vmNS) - netns.Set(vmNS) + err = netns.Set(vmNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } - log.Printf("Create the host vlan link after getting %s", client.eth0VethName) + log.Printf("Create the host vlan link after getting eth0: %s", client.eth0VethName) linkAttrs := vishnetlink.NewLinkAttrs() linkAttrs.Name = client.ethXVethName - //Get parent interface index. Is the index the same regardless of lib used? + //Get parent interface index. Index is consistent across libraries. eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) if err != nil { log.Printf("Failed to get interface: %s", client.eth0VethName) return newErrorNativeEndpointClient(err.Error()) } - //Is this how you set the peer? + //Set the peer linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ LinkAttrs: linkAttrs, VlanId: client.vlanID, } - err = vishnetlink.LinkAdd(link) - if err != nil { + if err = vishnetlink.LinkAdd(link); err != nil { return newErrorNativeEndpointClient(err.Error()) } log.Printf("Move vlan link to vnet NS: %d", uintptr(client.vnetNS)) if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNS)); err != nil { return newErrorNativeEndpointClient(err.Error()) } - vishnetlink.LinkSetUp(link) + if err = vishnetlink.LinkSetUp(link); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } log.Printf("Create veth pair (automatically set to UP)") if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } - log.Printf("Move vnetVethName into vnet namespace (peer will be moved in MoveEndpointsToContainerNS)") + log.Printf("Move vnetVethName into vnet namespace") if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNS)); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -160,7 +164,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { netns.Set(client.vnetNS) currNS, err := netns.Get() - log.Printf("Current NS after switch to vnet: %v.", currNS) + log.Printf("Current NS after switch to vnet: %s", currNS) if err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -178,7 +182,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return newErrorNativeEndpointClient(err.Error()) } client.vnetMac = vnetVethIf.HardwareAddr - //Set MTU? + log.Printf("Switch NS to vm") netns.Set(client.vmNS) @@ -194,8 +198,7 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error // Helper that creates routing rules for the current NS which direct packets // to the virtual gateway ip on linkToName device interface func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { - log.Printf("Add route for virtualgwip") - // ip route add 169.254.1.1/32 dev eth0 + log.Printf("Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0)") virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ Dst: *virtualGwNet, @@ -274,11 +277,20 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo if err = client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorNativeEndpointClient(err.Error()) } + log.Printf("Remove kernel subnet route automatically added by above") + for _, ipAddr := range epInfo.IPAddresses { + _, ipnet, _ := net.ParseCIDR(ipAddr.String()) + routeInfo := RouteInfo{ + Dst: *ipnet, + Scope: netlink.RT_SCOPE_LINK, + Protocol: netlink.RTPROT_KERNEL, + } + if err := deleteRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + return newErrorTransparentEndpointClient(err.Error()) + } + } - log.Printf("Container NS add route for virtualgwip") - // ip route add 169.254.1.1/32 dev eth0, but where do you specify eth0? - // addRoutes says it adds route to containerVethName, so this appears to be a route to the container? - //Currently called from container NS + log.Printf("Container NS add route for virtual gateway ip") if err := client.AddDefaultRoutes(client.containerVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -286,6 +298,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { return newErrorNativeEndpointClient(err.Error()) } + /* Ignore the resolv.conf for now log.Printf("Create resolv.conf for DNS") folder := fmt.Sprintf("/etc/netns/%s", client.vnetNSName) @@ -299,11 +312,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } */ - // ip route add dev - //Each endpoint only has two routes, which are the default. - //What is arp proxy? Set up arp rules here? - //AddEndpointRules vs. ConfigureContainerInterfacesAndRoutes currNS, err := netns.Get() log.Printf("Current NS before switch: %v.", currNS) if err != nil { @@ -336,10 +345,16 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } + log.Printf("Setting vnet loopback state to up") + err = client.netlink.SetLinkState(loopbackIf, true) + if err != nil { + log.Printf("Failed to set loopback link state to up") + return newErrorNativeEndpointClient(err.Error()) + } + var routeInfoList []RouteInfo - // ip route add dev - // This route is needed for incoming packets to pod to route via hostveth + // Add route specifying which device the pod ip(s) are on for _, ipAddr := range epInfo.IPAddresses { var ( routeInfo RouteInfo From 3f6507d498a0e7a62ee683cdf43b4862a18a8f36 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 8 Jul 2022 15:00:28 -0700 Subject: [PATCH 07/86] Cannot set link to up if it's in another NS --- network/native_endpointclient_linux.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 1f6a666786..7531375bf4 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -115,6 +115,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { LinkAttrs: linkAttrs, VlanId: client.vlanID, } + log.Printf("Add link to VM NS (automatically set to UP)") if err = vishnetlink.LinkAdd(link); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -122,9 +123,6 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNS)); err != nil { return newErrorNativeEndpointClient(err.Error()) } - if err = vishnetlink.LinkSetUp(link); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } log.Printf("Create veth pair (automatically set to UP)") if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { From d38d06b98e155c3bccc5c5ba7ca6cb931dd320cc Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 11 Jul 2022 12:04:16 -0700 Subject: [PATCH 08/86] Multiple containers on same VNET NS --- network/native_endpointclient_linux.go | 31 +++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 7531375bf4..21ee5a8234 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net" + "strings" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" @@ -89,9 +90,14 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { log.Printf("Save VM namespace: %s", vmNS) client.vmNS = vmNS - log.Printf("Create the vnet namespace and switch to it") - if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { - return newErrorNativeEndpointClient(err.Error()) + log.Printf("Checking if NS exists...") + var existingErr error + client.vnetNS, existingErr = netns.GetFromName(client.vnetNSName) + if existingErr != nil { + log.Printf("No existing NS detected. Creating the vnet namespace and switching to it") + if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } } log.Printf("Set current namespace to VM: %s", vmNS) @@ -116,12 +122,21 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { VlanId: client.vlanID, } log.Printf("Add link to VM NS (automatically set to UP)") - if err = vishnetlink.LinkAdd(link); err != nil { - return newErrorNativeEndpointClient(err.Error()) + existingErr = vishnetlink.LinkAdd(link) + ethXCreated := true + if existingErr != nil { + if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { + return newErrorNativeEndpointClient(err.Error()) + } else { + log.Printf("eth0.X exists") + ethXCreated = false + } } - log.Printf("Move vlan link to vnet NS: %d", uintptr(client.vnetNS)) - if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNS)); err != nil { - return newErrorNativeEndpointClient(err.Error()) + if ethXCreated { + log.Printf("Move vlan link (ethX) to vnet NS: %d", uintptr(client.vnetNS)) + if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNS)); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } } log.Printf("Create veth pair (automatically set to UP)") From 1f8d37f706f34fa4f5925ad2a1b747f4f9979d9d Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 11 Jul 2022 16:09:57 -0700 Subject: [PATCH 09/86] Delete Endpoint routes on Delete --- network/native_endpointclient_linux.go | 79 ++++++++++++++++++++------ 1 file changed, 62 insertions(+), 17 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 21ee5a8234..5385a74e62 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -257,8 +257,8 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac } return nil } -func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { - +func (client *NativeEndpointClient) DeleteEndpointRules(ep *EndpointInfo) { + //Never added any endpoint rules } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { log.Printf("Moving endpoint to container NS") @@ -276,6 +276,27 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf return nil } +func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []RouteInfo { + var routeInfoList []RouteInfo + // Add route specifying which device the pod ip(s) are on + for _, ipAddr := range ipAddresses { + var ( + routeInfo RouteInfo + ipNet net.IPNet + ) + + if ipAddr.IP.To4() != nil { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} + } else { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} + } + log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) + routeInfo.Dst = ipNet + routeInfoList = append(routeInfoList, routeInfo) + + } + return routeInfoList +} func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { log.Printf("Setting NS to container path %d", uintptr(client.vnetNS)) contNS, err := netns.GetFromPath(epInfo.NetNsPath) @@ -368,22 +389,8 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo var routeInfoList []RouteInfo // Add route specifying which device the pod ip(s) are on - for _, ipAddr := range epInfo.IPAddresses { - var ( - routeInfo RouteInfo - ipNet net.IPNet - ) - - if ipAddr.IP.To4() != nil { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} - } else { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} - } - log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) - routeInfo.Dst = ipNet - routeInfoList = append(routeInfoList, routeInfo) + routeInfoList = client.GetVnetRoutes(epInfo.IPAddresses) - } log.Printf("Client data: ethX: %s, vnet: %s", client.ethXVethName, client.vnetVethName) log.Printf("Vnet NS add default/gateway routes (Assuming indempotent)") @@ -408,5 +415,43 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return nil } func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { + // Open the vnet network namespace + log.Printf("Opening vnetns %v.", fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) + ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) + if err != nil { + return err + } + defer ns.Close() + // Enter the vnet network namespace + log.Printf("Entering vnetns %v.", ns) + if err := ns.Enter(); err != nil { + return err + } + + // Exit vnet network namespace + defer func() { + log.Printf("Exiting vnetns %v.", ns) + if err := ns.Exit(); err != nil { + log.Printf("Could not exit vnetns, err:%v.", err) + } + }() + log.Printf("Enter vnet NS") + err = netns.Set(client.vnetNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + log.Printf("Removing routes") + var routeInfoList []RouteInfo + routeInfoList = client.GetVnetRoutes(ep.IPAddresses) + if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + log.Printf("Leave vnet NS") + err = netns.Set(client.vmNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } return nil } From 7f8b80225ee51377dbd8aa17010a06474cf81a7b Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 11 Jul 2022 16:48:15 -0700 Subject: [PATCH 10/86] Minimizing netns usage --- network/native_endpointclient_linux.go | 76 +++----------------------- 1 file changed, 8 insertions(+), 68 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 5385a74e62..21fac158e3 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -126,7 +126,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { ethXCreated := true if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient(existingErr.Error()) } else { log.Printf("eth0.X exists") ethXCreated = false @@ -152,18 +152,9 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { defer func() { if err != nil { log.Printf("Switching NS to vnet") - netns.Set(client.vnetNS) + log.Printf("Failure detected, deleting links...") - //Delete vnet <-> container - if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { - log.Errorf("Deleting vnetVeth failed on addendpoint failure:%v", delErr) - } - //Delete eth0 <-> eth0.X - if delErr := client.netlink.DeleteLink(client.ethXVethName); delErr != nil { - log.Errorf("Deleting hostVeth failed on addendpoint failure:%v", delErr) - } - log.Printf("Switching NS to vm") - netns.Set(client.vmNS) + } }() log.Printf("Check that container veth exists.") @@ -257,7 +248,7 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac } return nil } -func (client *NativeEndpointClient) DeleteEndpointRules(ep *EndpointInfo) { +func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { //Never added any endpoint rules } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { @@ -299,16 +290,9 @@ func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []Rou } func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { log.Printf("Setting NS to container path %d", uintptr(client.vnetNS)) - contNS, err := netns.GetFromPath(epInfo.NetNsPath) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - err = netns.Set(contNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } + log.Printf("Assign IPs to container veth interface") - if err = client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorNativeEndpointClient(err.Error()) } log.Printf("Remove kernel subnet route automatically added by above") @@ -333,26 +317,6 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } - /* Ignore the resolv.conf for now - log.Printf("Create resolv.conf for DNS") - folder := fmt.Sprintf("/etc/netns/%s", client.vnetNSName) - resolv := fmt.Sprintf("%s/resolv.conf", folder) - if _, err := os.Stat(folder); os.IsNotExist(err) { - os.MkdirAll(folder, 0700) // Create your file - } - log.Printf("Writing to resolv.conf file %s , %s", epInfo.DNS.Servers, epInfo.DNS.Servers[0]) - data := []byte(fmt.Sprintf("nameserver %s", epInfo.DNS.Servers[0])) - if err := os.WriteFile(resolv, data, 0644); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - */ - - currNS, err := netns.Get() - log.Printf("Current NS before switch: %v.", currNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - // Open the vnet network namespace log.Printf("Opening vnetns %v.", fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) @@ -374,11 +338,6 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo } }() - log.Printf("Current NS after switch: %v.", currNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - log.Printf("Setting vnet loopback state to up") err = client.netlink.SetLinkState(loopbackIf, true) if err != nil { @@ -386,10 +345,8 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } - var routeInfoList []RouteInfo - // Add route specifying which device the pod ip(s) are on - routeInfoList = client.GetVnetRoutes(epInfo.IPAddresses) + routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) log.Printf("Client data: ethX: %s, vnet: %s", client.ethXVethName, client.vnetVethName) @@ -406,12 +363,6 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } - log.Printf("Return to container NS") - err = netns.Set(contNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - return nil } func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { @@ -435,23 +386,12 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { log.Printf("Could not exit vnetns, err:%v.", err) } }() - log.Printf("Enter vnet NS") - err = netns.Set(client.vnetNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } log.Printf("Removing routes") - var routeInfoList []RouteInfo - routeInfoList = client.GetVnetRoutes(ep.IPAddresses) + routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return newErrorNativeEndpointClient(err.Error()) } - log.Printf("Leave vnet NS") - err = netns.Set(client.vmNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } return nil } From b62204986ef92ff9b4d5e3677ea07186bf847900 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 12 Jul 2022 10:28:53 -0700 Subject: [PATCH 11/86] Moving NS Exec Code --- network/native_endpointclient_linux.go | 56 ++++++++++---------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 21fac158e3..d1474dcd67 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -317,29 +317,12 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } - // Open the vnet network namespace - log.Printf("Opening vnetns %v.", fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) - ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) - if err != nil { - return err - } - defer ns.Close() - // Enter the vnet network namespace - log.Printf("Entering vnetns %v.", ns) - if err := ns.Enter(); err != nil { - return err - } - - // Exit vnet network namespace - defer func() { - log.Printf("Exiting vnetns %v.", ns) - if err := ns.Exit(); err != nil { - log.Printf("Could not exit vnetns, err:%v.", err) - } - }() - + err := ExecuteInNS(client.vnetNSName, epInfo, client.ConfigureVnetInterfacesAndRoutes) + return err +} +func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutes(epInfo *EndpointInfo) error { log.Printf("Setting vnet loopback state to up") - err = client.netlink.SetLinkState(loopbackIf, true) + err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { log.Printf("Failed to set loopback link state to up") return newErrorNativeEndpointClient(err.Error()) @@ -362,13 +345,25 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return newErrorNativeEndpointClient(err.Error()) } + return err +} +func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { + return ExecuteInNS(client.vnetNSName, ep, client.DeleteEndpointsImpl) +} +func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { + log.Printf("Removing routes") + routeInfoList := client.GetVnetRoutes(ep.IPAddresses) + if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } return nil } -func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { + +func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { // Open the vnet network namespace - log.Printf("Opening vnetns %v.", fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) - ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) + log.Printf("Opening ns %v.", fmt.Sprintf("/var/run/netns/%s", nsName)) + ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", nsName)) if err != nil { return err } @@ -383,15 +378,8 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { defer func() { log.Printf("Exiting vnetns %v.", ns) if err := ns.Exit(); err != nil { - log.Printf("Could not exit vnetns, err:%v.", err) + log.Printf("Could not exit ns, err:%v.", err) } }() - - log.Printf("Removing routes") - routeInfoList := client.GetVnetRoutes(ep.IPAddresses) - if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - - return nil + return f(param) } From c1e60a1f009fd2616e04e96031a5ac8775648520 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 12 Jul 2022 11:22:06 -0700 Subject: [PATCH 12/86] Further minimized netns.Set usage --- network/native_endpointclient_linux.go | 55 ++++++++++++++++---------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index d1474dcd67..ad6ab1674f 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -87,6 +87,15 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if err != nil { return newErrorNativeEndpointClient(err.Error()) } + //Mostly for reference + returnedTo, err := GetCurrentThreadNamespace() + if err != nil { + log.Printf("Unable to get VM namespace: %v", err) + return newErrorNativeEndpointClient(err.Error()) + } else { + log.Printf("VM Namespace: %v", returnedTo) + } + log.Printf("Save VM namespace: %s", vmNS) client.vmNS = vmNS @@ -148,15 +157,6 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return newErrorNativeEndpointClient(err.Error()) } - //If there is a failure, delete the links - defer func() { - if err != nil { - log.Printf("Switching NS to vnet") - - log.Printf("Failure detected, deleting links...") - - } - }() log.Printf("Check that container veth exists.") containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) if err != nil { @@ -164,8 +164,14 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } client.containerMac = containerIf.HardwareAddr - log.Printf("Switch NS to vnet") - netns.Set(client.vnetNS) + err = ExecuteInNS(client.vnetNSName, epInfo, client.PopulateVnet) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + return nil +} +func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { currNS, err := netns.Get() log.Printf("Current NS after switch to vnet: %s", currNS) @@ -186,10 +192,6 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return newErrorNativeEndpointClient(err.Error()) } client.vnetMac = vnetVethIf.HardwareAddr - - log.Printf("Switch NS to vm") - netns.Set(client.vmNS) - return nil } func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { @@ -289,7 +291,6 @@ func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []Rou return routeInfoList } func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - log.Printf("Setting NS to container path %d", uintptr(client.vnetNS)) log.Printf("Assign IPs to container veth interface") if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { @@ -361,24 +362,38 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { } func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { + // Current namespace + returnedTo, err := GetCurrentThreadNamespace() + if err != nil { + log.Printf("[ExecuteInNS] Could not get NS we are in: %v", err) + } else { + log.Printf("[ExecuteInNS] In NS before switch: %v", returnedTo) + } + // Open the vnet network namespace - log.Printf("Opening ns %v.", fmt.Sprintf("/var/run/netns/%s", nsName)) + log.Printf("[ExecuteInNS] Opening ns %v.", fmt.Sprintf("/var/run/netns/%s", nsName)) ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", nsName)) if err != nil { return err } defer ns.Close() // Enter the vnet network namespace - log.Printf("Entering vnetns %v.", ns) + log.Printf("[ExecuteInNS] Entering vnetns %v.", ns) if err := ns.Enter(); err != nil { return err } // Exit vnet network namespace defer func() { - log.Printf("Exiting vnetns %v.", ns) + log.Printf("[ExecuteInNS] Exiting vnetns %v.", ns) if err := ns.Exit(); err != nil { - log.Printf("Could not exit ns, err:%v.", err) + log.Printf("[ExecuteInNS] Could not exit ns, err:%v.", err) + } + returnedTo, err := GetCurrentThreadNamespace() + if err != nil { + log.Printf("[ExecuteInNS] Could not get NS we returned to: %v", err) + } else { + log.Printf("[ExecuteInNS] Returned to NS: %v", returnedTo) } }() return f(param) From ddd5213464c196832ce123bc98b7fa5d9f15e629 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 12 Jul 2022 16:00:58 -0700 Subject: [PATCH 13/86] Moved helper methods down, drafted tests --- network/native_endpointclient_linux.go | 177 ++++++++-------- network/native_endpointclient_linux_test.go | 222 ++++++++++++++++++++ 2 files changed, 315 insertions(+), 84 deletions(-) create mode 100644 network/native_endpointclient_linux_test.go diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index ad6ab1674f..38093a30e0 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -93,7 +93,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { log.Printf("Unable to get VM namespace: %v", err) return newErrorNativeEndpointClient(err.Error()) } else { - log.Printf("VM Namespace: %v", returnedTo) + log.Printf("VM Namespace: %s", returnedTo.file.Name()) } log.Printf("Save VM namespace: %s", vmNS) @@ -102,10 +102,15 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { log.Printf("Checking if NS exists...") var existingErr error client.vnetNS, existingErr = netns.GetFromName(client.vnetNSName) + //If the ns does not exist, the below code will trigger to create it if existingErr != nil { - log.Printf("No existing NS detected. Creating the vnet namespace and switching to it") - if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { - return newErrorNativeEndpointClient(err.Error()) + if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { + return newErrorNativeEndpointClient(existingErr.Error()) + } else { + log.Printf("No existing NS detected. Creating the vnet namespace and switching to it") + if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } } } @@ -137,7 +142,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { return newErrorNativeEndpointClient(existingErr.Error()) } else { - log.Printf("eth0.X exists") + log.Printf("eth0.X already exists") ethXCreated = false } } @@ -166,7 +171,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { err = ExecuteInNS(client.vnetNSName, epInfo, client.PopulateVnet) if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return err } return nil @@ -201,55 +206,6 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error return nil } -// Helper that creates routing rules for the current NS which direct packets -// to the virtual gateway ip on linkToName device interface -func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { - log.Printf("Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0)") - virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) - routeInfo := RouteInfo{ - Dst: *virtualGwNet, - Scope: netlink.RT_SCOPE_LINK, - DevName: linkToName, - } - if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { - return err - } - - log.Printf("Add default route (ip route add default via 169.254.1.1 dev eth0)") - _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) - dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} - routeInfo = RouteInfo{ - Dst: dstIP, - Gw: virtualGwIP, - DevName: linkToName, - } - - if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { - return err - } - return nil -} - -// Helper that creates arp entry for the current NS which maps the virtual -// gateway to destMac -func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { - _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) - // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac - log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", - virtualGwNet.String(), destMac) - hardwareAddr, err := net.ParseMAC(destMac) - if err != nil { - return err - } - if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, - interfaceName, //What is the purpose of name? - virtualGwNet.IP, - hardwareAddr, - false); err != nil { - return fmt.Errorf("adding arp entry failed: %w", err) - } - return nil -} func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { //Never added any endpoint rules } @@ -269,27 +225,7 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf return nil } -func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []RouteInfo { - var routeInfoList []RouteInfo - // Add route specifying which device the pod ip(s) are on - for _, ipAddr := range ipAddresses { - var ( - routeInfo RouteInfo - ipNet net.IPNet - ) - - if ipAddr.IP.To4() != nil { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} - } else { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} - } - log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) - routeInfo.Dst = ipNet - routeInfoList = append(routeInfoList, routeInfo) - } - return routeInfoList -} func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { log.Printf("Assign IPs to container veth interface") @@ -317,7 +253,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { return newErrorNativeEndpointClient(err.Error()) } - + //Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes err := ExecuteInNS(client.vnetNSName, epInfo, client.ConfigureVnetInterfacesAndRoutes) return err } @@ -342,13 +278,86 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutes(epInfo *End if err = client.AddDefaultArp(client.ethXVethName, azureMac); err != nil { return newErrorNativeEndpointClient(err.Error()) } - log.Printf("Adding routes") + log.Printf("Adding routes to vnet specific to this container") if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return newErrorNativeEndpointClient(err.Error()) } + // Return to ConfigureContainerInterfacesAndRoutes return err } +// Helper that gets the routes in the vnet NS for a particular list of IP addresses +func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []RouteInfo { + var routeInfoList []RouteInfo + // Add route specifying which device the pod ip(s) are on + for _, ipAddr := range ipAddresses { + var ( + routeInfo RouteInfo + ipNet net.IPNet + ) + + if ipAddr.IP.To4() != nil { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} + } else { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} + } + log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) + routeInfo.Dst = ipNet + routeInfoList = append(routeInfoList, routeInfo) + + } + return routeInfoList +} + +// Helper that creates routing rules for the current NS which direct packets +// to the virtual gateway ip on linkToName device interface +func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { + log.Printf("Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0)") + virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) + routeInfo := RouteInfo{ + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + DevName: linkToName, + } + if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { + return err + } + + log.Printf("Add default route (ip route add default via 169.254.1.1 dev eth0)") + _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) + dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} + routeInfo = RouteInfo{ + Dst: dstIP, + Gw: virtualGwIP, + DevName: linkToName, + } + + if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { + return err + } + return nil +} + +// Helper that creates arp entry for the current NS which maps the virtual +// gateway to destMac +func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { + _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) + // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac + log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", + virtualGwNet.String(), destMac) + hardwareAddr, err := net.ParseMAC(destMac) + if err != nil { + return err + } + if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, + interfaceName, //What is the purpose of name? + virtualGwNet.IP, + hardwareAddr, + false); err != nil { + return fmt.Errorf("adding arp entry failed: %w", err) + } + return nil +} func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { return ExecuteInNS(client.vnetNSName, ep, client.DeleteEndpointsImpl) } @@ -367,25 +376,25 @@ func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { if err != nil { log.Printf("[ExecuteInNS] Could not get NS we are in: %v", err) } else { - log.Printf("[ExecuteInNS] In NS before switch: %v", returnedTo) + log.Printf("[ExecuteInNS] In NS before switch: %s", returnedTo.file.Name()) } - // Open the vnet network namespace + // Open the network namespace log.Printf("[ExecuteInNS] Opening ns %v.", fmt.Sprintf("/var/run/netns/%s", nsName)) ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", nsName)) if err != nil { return err } defer ns.Close() - // Enter the vnet network namespace - log.Printf("[ExecuteInNS] Entering vnetns %v.", ns) + // Enter the network namespace + log.Printf("[ExecuteInNS] Entering vnetns %s.", ns.file.Name()) if err := ns.Enter(); err != nil { return err } - // Exit vnet network namespace + // Exit network namespace defer func() { - log.Printf("[ExecuteInNS] Exiting vnetns %v.", ns) + log.Printf("[ExecuteInNS] Exiting vnetns %s.", ns.file.Name()) if err := ns.Exit(); err != nil { log.Printf("[ExecuteInNS] Could not exit ns, err:%v.", err) } @@ -393,7 +402,7 @@ func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { if err != nil { log.Printf("[ExecuteInNS] Could not get NS we returned to: %v", err) } else { - log.Printf("[ExecuteInNS] Returned to NS: %v", returnedTo) + log.Printf("[ExecuteInNS] Returned to NS: %s", returnedTo.file.Name()) } }() return f(param) diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go new file mode 100644 index 0000000000..22e26b3dd6 --- /dev/null +++ b/network/native_endpointclient_linux_test.go @@ -0,0 +1,222 @@ +//go:build linux +// +build linux + +package network + +import ( + "net" + "testing" + + "github.com/Azure/azure-container-networking/netio" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/platform" + "github.com/stretchr/testify/require" +) + +func TestNativeAddEndpoints(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + tests := []struct { + name string + client *NativeEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Add endpoints", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: false, + }, + { + name: "Add endpoints netlink fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + netlink.ErrorMockNetlink.Error() + " : netlink fail", + }, + { + name: "Add endpoints get interface fail for primary interface", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 1), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":eth0", + }, + { + name: "Add endpoints get interface fail for getting container veth", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 2), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + }, + { + name: "get interface fail for ethX veth", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 3), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":eth0.1", + }, + { + name: "get interface fail for vnet veth", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 4), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":A1veth0", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.AddEndpoints(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + vnetMac, _ := net.ParseMAC("ab:cd:ef:12:34:56") + + tests := []struct { + name string + client *NativeEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Configure Interface and routes good path", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + vnetMac: vnetMac, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: false, + }, + { + name: "Configure Interface and routes assign ip fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + vnetMac: vnetMac, + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: "netlink fail", + }, + // Any failure of netioshim causes problems + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.ConfigureContainerInterfacesAndRoutes(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} From f4b5fd7b7aaf9c127c7a12d7d30d999f4dfe6104 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 12 Jul 2022 16:49:27 -0700 Subject: [PATCH 14/86] Removed DevName from Route Info, more tests --- network/native_endpointclient_linux.go | 13 +++--- network/native_endpointclient_linux_test.go | 50 +++++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 38093a30e0..d27a88a547 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -315,10 +315,11 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { log.Printf("Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0)") virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ - Dst: *virtualGwNet, - Scope: netlink.RT_SCOPE_LINK, - DevName: linkToName, + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + //DevName: linkToName, } + // Difference between interface name in addRoutes and DevName: in RouteInfo? if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { return err } @@ -327,9 +328,9 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} routeInfo = RouteInfo{ - Dst: dstIP, - Gw: virtualGwIP, - DevName: linkToName, + Dst: dstIP, + Gw: virtualGwIP, + //DevName: linkToName, } if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 22e26b3dd6..637643cbc3 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -204,6 +204,56 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErr: true, wantErrMsg: "netlink fail", }, + { + name: "Configure Interface and routes container final route (2nd default route) fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + vnetMac: vnetMac, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 3), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + }, + { + name: "Configure Interface and routes vnet final route (specific to container) fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + vnetMac: vnetMac, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 6), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", + }, // Any failure of netioshim causes problems } From 4c280c22e4077b8c73e0b94ffbb8f04cb55986ef Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 13 Jul 2022 10:44:48 -0700 Subject: [PATCH 15/86] Test existing vnet ns, delete endpoint --- network/native_endpointclient_linux.go | 7 +- network/native_endpointclient_linux_test.go | 89 ++++++++++++++++++++- 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index d27a88a547..72b699e9c9 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -317,7 +317,6 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { routeInfo := RouteInfo{ Dst: *virtualGwNet, Scope: netlink.RT_SCOPE_LINK, - //DevName: linkToName, } // Difference between interface name in addRoutes and DevName: in RouteInfo? if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { @@ -330,7 +329,6 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { routeInfo = RouteInfo{ Dst: dstIP, Gw: virtualGwIP, - //DevName: linkToName, } if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { @@ -340,7 +338,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { } // Helper that creates arp entry for the current NS which maps the virtual -// gateway to destMac +// gateway to destMac on a particular interfaceName func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac @@ -351,7 +349,7 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac return err } if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, - interfaceName, //What is the purpose of name? + interfaceName, virtualGwNet.IP, hardwareAddr, false); err != nil { @@ -371,6 +369,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { return nil } +// Helper function that allows executing a function with one parameter in a VM namespace func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { // Current namespace returnedTo, err := GetCurrentThreadNamespace() diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 637643cbc3..b34f53bb11 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -41,6 +41,22 @@ func TestNativeAddEndpoints(t *testing.T) { epInfo: &EndpointInfo{}, wantErr: false, }, + { + name: "Add endpoints same vnet", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth1", + containerVethName: "B2veth1", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: false, + }, { name: "Add endpoints netlink fail", client: &NativeEndpointClient{ @@ -141,6 +157,46 @@ func TestNativeAddEndpoints(t *testing.T) { }) } } +func TestNativeDeleteEndpoints(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + tests := []struct { + name string + client *NativeEndpointClient + ep *endpoint + }{ + { + name: "Delete endpoint rules good path", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + ep: &endpoint{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + tt.client.DeleteEndpoints(tt.ep) + }) + } +} func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { nl := netlink.NewMockNetlink(false, "") @@ -179,6 +235,38 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, wantErr: false, }, + { + name: "Configure Interface and routes multiple IPs", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + vnetMac: vnetMac, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + { + IP: net.ParseIP("192.168.0.6"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + { + IP: net.ParseIP("192.168.0.8"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: false, + }, { name: "Configure Interface and routes assign ip fail", client: &NativeEndpointClient{ @@ -254,7 +342,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErr: true, wantErrMsg: "NativeEndpointClient Error : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", }, - // Any failure of netioshim causes problems } for _, tt := range tests { From 7ac1a953155a344fc7bb36aae5a193a60edca07c Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 13 Jul 2022 12:09:31 -0700 Subject: [PATCH 16/86] NetNS interface for testing --- network/native_endpointclient_linux.go | 102 +++++++++++++++++++++---- 1 file changed, 87 insertions(+), 15 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 72b699e9c9..0bead7b917 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -21,6 +21,78 @@ const ( loopbackIf = "lo" ) +//Move somewhere else +type NetnsInterface interface { + Get() (fileDescriptor uintptr, err error) + GetFromName(name string) (fileDescriptor uintptr, err error) + Set(fileDescriptor uintptr) (err error) + NewNamed(name string) (fileDescriptor uintptr, err error) +} +type Netns struct{} + +func NewNetns() *Netns { + return &Netns{} +} +func (f *Netns) Get() (uintptr, error) { + nsHandle, err := netns.Get() + return uintptr(nsHandle), err +} +func (f *Netns) GetFromName(name string) (uintptr, error) { + nsHandle, err := netns.GetFromName(name) + return uintptr(nsHandle), err +} +func (f *Netns) Set(fileDescriptor uintptr) error { + return netns.Set(netns.NsHandle(fileDescriptor)) +} +func (f *Netns) NewNamed(name string) (uintptr, error) { + nsHandle, err := netns.NewNamed(name) + return uintptr(nsHandle), err +} + +var ErrorMockNetns = errors.New("mock netns error") + +func newErrorMockNetns(errStr string) error { + return fmt.Errorf("%w : %s", ErrorMockNetns, errStr) +} + +type MockNetns struct { + failMethod int + failMessage string +} + +func NewMockNetns(failMethod int, failMessage string) *MockNetns { + return &MockNetns{ + failMethod: failMethod, + failMessage: failMessage, + } +} +func (f *MockNetns) Get() (uintptr, error) { + if f.failMethod == 1 { + return 0, newErrorMockNetns(f.failMessage) + } + return 1, nil +} +func (f *MockNetns) GetFromName(name string) (uintptr, error) { + if f.failMethod == 2 { + return 0, newErrorMockNetns(f.failMessage) + } + return 1, nil +} +func (f *MockNetns) Set(handle uintptr) error { + if f.failMethod == 3 { + return newErrorMockNetns(f.failMessage) + } + return nil +} +func (f *MockNetns) NewNamed(name string) (uintptr, error) { + if f.failMethod == 4 { + return 0, newErrorMockNetns(f.failMessage) + } + return 1, nil +} + +//End move somewhere else + var errorNativeEndpointClient = errors.New("NativeEndpointClient Error") func newErrorNativeEndpointClient(errStr string) error { @@ -37,13 +109,12 @@ type NativeEndpointClient struct { containerMac net.HardwareAddr ethXMac net.HardwareAddr - vmNS netns.NsHandle - - vnetNSName string - vnetNS netns.NsHandle + vnetNSName string + vnetNSFileDescriptor uintptr mode string vlanID int + netnsClient NetnsInterface netlink netlink.NetlinkInterface netioshim netio.NetIOInterface plClient platform.ExecClient @@ -71,6 +142,7 @@ func NewNativeEndpointClient( vnetNSName: vnetNSName, mode: mode, vlanID: vlanid, + netnsClient: NewNetns(), netlink: nl, netioshim: &netio.NetIO{}, plClient: plc, @@ -83,7 +155,7 @@ func NewNativeEndpointClient( func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { var err error log.Printf("Get VM namespace handle") - vmNS, err := netns.Get() + vmNS, err := client.netnsClient.Get() if err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -96,26 +168,26 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { log.Printf("VM Namespace: %s", returnedTo.file.Name()) } - log.Printf("Save VM namespace: %s", vmNS) - client.vmNS = vmNS - log.Printf("Checking if NS exists...") var existingErr error - client.vnetNS, existingErr = netns.GetFromName(client.vnetNSName) + vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) //If the ns does not exist, the below code will trigger to create it if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { return newErrorNativeEndpointClient(existingErr.Error()) } else { log.Printf("No existing NS detected. Creating the vnet namespace and switching to it") - if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { + vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) + if err != nil { return newErrorNativeEndpointClient(err.Error()) } + } } + client.vnetNSFileDescriptor = uintptr(vnetNS) log.Printf("Set current namespace to VM: %s", vmNS) - err = netns.Set(vmNS) + err = client.netnsClient.Set(vmNS) if err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -147,8 +219,8 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } } if ethXCreated { - log.Printf("Move vlan link (ethX) to vnet NS: %d", uintptr(client.vnetNS)) - if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNS)); err != nil { + log.Printf("Move vlan link (ethX) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) + if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { return newErrorNativeEndpointClient(err.Error()) } } @@ -158,7 +230,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return newErrorNativeEndpointClient(err.Error()) } log.Printf("Move vnetVethName into vnet namespace") - if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNS)); err != nil { + if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -178,7 +250,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { - currNS, err := netns.Get() + currNS, err := client.netnsClient.Get() log.Printf("Current NS after switch to vnet: %s", currNS) if err != nil { return newErrorNativeEndpointClient(err.Error()) From 32b4d98579a6abb346520d0f4c487d9b1a3cc701 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 13 Jul 2022 14:57:55 -0700 Subject: [PATCH 17/86] Separated tests by namespace --- network/native_endpointclient_linux.go | 43 +++++-- network/native_endpointclient_linux_test.go | 131 ++++++++++++++++---- 2 files changed, 139 insertions(+), 35 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 0bead7b917..223653cc87 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -154,6 +154,21 @@ func NewNativeEndpointClient( func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { var err error + err = client.PopulateClient(epInfo) + if err != nil { + return err + } + + err = ExecuteInNS(client.vnetNSName, epInfo, client.PopulateVnet) + if err != nil { + return err + } + + return nil +} + +// Called from AddEndpoints, Namespace: VM +func (client *NativeEndpointClient) PopulateClient(epInfo *EndpointInfo) error { log.Printf("Get VM namespace handle") vmNS, err := client.netnsClient.Get() if err != nil { @@ -240,14 +255,10 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return newErrorNativeEndpointClient(err.Error()) } client.containerMac = containerIf.HardwareAddr - - err = ExecuteInNS(client.vnetNSName, epInfo, client.PopulateVnet) - if err != nil { - return err - } - return nil } + +// Called from AddEndpoints, Namespace: Vnet func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { currNS, err := client.netnsClient.Get() @@ -299,7 +310,18 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf } func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + err := client.ConfigureContainerInterfacesAndRoutesImpl(epInfo) + if err != nil { + return err + } + //Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes + err = ExecuteInNS(client.vnetNSName, epInfo, client.ConfigureVnetInterfacesAndRoutesImpl) + return err +} + +// Called from ConfigureContainerInterfacesAndRoutes, Namespace: Container +func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { log.Printf("Assign IPs to container veth interface") if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorNativeEndpointClient(err.Error()) @@ -325,11 +347,11 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { return newErrorNativeEndpointClient(err.Error()) } - //Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes - err := ExecuteInNS(client.vnetNSName, epInfo, client.ConfigureVnetInterfacesAndRoutes) - return err + return nil } -func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutes(epInfo *EndpointInfo) error { + +// Called from ConfigureContainerInterfacesAndRoutes, Namespace: Vnet +func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { log.Printf("Setting vnet loopback state to up") err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { @@ -438,6 +460,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return newErrorNativeEndpointClient(err.Error()) } + return nil } diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index b34f53bb11..7984c5cfcd 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -33,22 +33,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), - }, - epInfo: &EndpointInfo{}, - wantErr: false, - }, - { - name: "Add endpoints same vnet", - client: &NativeEndpointClient{ - eth0VethName: "eth0", - ethXVethName: "eth0.1", - vnetVethName: "A1veth1", - containerVethName: "B2veth1", - vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -65,6 +50,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -82,6 +68,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -99,6 +86,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -108,35 +96,76 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr: true, wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.PopulateClient(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } + + tests = []struct { + name string + client *NativeEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ { - name: "get interface fail for ethX veth", + name: "Add endpoints second half", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 3), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: false, + }, + { + name: "Add endpoints netlink fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, wantErr: true, wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":eth0.1", }, { - name: "get interface fail for vnet veth", + name: "Add endpoints netlink fail", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 4), + netioshim: netio.NewMockNetIO(true, 2), }, epInfo: &EndpointInfo{}, wantErr: true, @@ -147,7 +176,7 @@ func TestNativeAddEndpoints(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.AddEndpoints(tt.epInfo) + err := tt.client.PopulateVnet(tt.epInfo) if tt.wantErr { require.Error(t, err) require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) @@ -174,6 +203,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -193,7 +223,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - tt.client.DeleteEndpoints(tt.ep) + tt.client.DeleteEndpointsImpl(tt.ep) }) } } @@ -220,6 +250,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -244,6 +275,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -276,6 +308,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -301,6 +334,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -317,8 +351,54 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErr: true, wantErrMsg: "NativeEndpointClient Error : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.ConfigureContainerInterfacesAndRoutesImpl(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } + tests = []struct { + name string + client *NativeEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Configure Interface and routes good path second half", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + vnetMac: vnetMac, + netnsClient: NewMockNetns(0, ""), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: false, + }, { - name: "Configure Interface and routes vnet final route (specific to container) fail", + name: "Configure Interface and routes final routes for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -326,10 +406,11 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 6), + netioshim: netio.NewMockNetIO(true, 3), }, epInfo: &EndpointInfo{ IPAddresses: []net.IPNet{ @@ -347,7 +428,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.ConfigureContainerInterfacesAndRoutes(tt.epInfo) + err := tt.client.ConfigureVnetInterfacesAndRoutesImpl(tt.epInfo) if tt.wantErr { require.Error(t, err) require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) From 9a488f9674363087d6a9cfa0cb51354e35ea1b48 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 14 Jul 2022 16:08:24 -0700 Subject: [PATCH 18/86] Endpoints delete if they cannot be moved into NS --- network/native_endpointclient_linux.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 223653cc87..5478cb7cb6 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -152,6 +152,7 @@ func NewNativeEndpointClient( return client } +// Adds interfaces to the vnet (created if not existing) and vm namespace func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { var err error err = client.PopulateClient(epInfo) @@ -223,6 +224,7 @@ func (client *NativeEndpointClient) PopulateClient(epInfo *EndpointInfo) error { VlanId: client.vlanID, } log.Printf("Add link to VM NS (automatically set to UP)") + // Attempting to create ethX existingErr = vishnetlink.LinkAdd(link) ethXCreated := true if existingErr != nil { @@ -236,6 +238,10 @@ func (client *NativeEndpointClient) PopulateClient(epInfo *EndpointInfo) error { if ethXCreated { log.Printf("Move vlan link (ethX) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { + log.Printf("Deleting ethX veth in VM NS due to addendpoint failure") + if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { + log.Errorf("Deleting ethX veth failed on addendpoint failure:%v", delErr) + } return newErrorNativeEndpointClient(err.Error()) } } @@ -244,8 +250,13 @@ func (client *NativeEndpointClient) PopulateClient(epInfo *EndpointInfo) error { if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } + log.Printf("Move vnetVethName into vnet namespace") if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { + log.Printf("Deleting vnet veth due to addendpoint failure") + if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { + log.Errorf("Deleting vnet veth failed on addendpoint failure:%v", delErr) + } return newErrorNativeEndpointClient(err.Error()) } @@ -309,6 +320,7 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf return nil } +// Adds routes, arp entries, etc. to the vnet and container namespaces func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { err := client.ConfigureContainerInterfacesAndRoutesImpl(epInfo) if err != nil { @@ -381,6 +393,7 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo } // Helper that gets the routes in the vnet NS for a particular list of IP addresses +// Example: 192.168.0.4 dev proto static func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []RouteInfo { var routeInfoList []RouteInfo // Add route specifying which device the pod ip(s) are on @@ -405,6 +418,8 @@ func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []Rou // Helper that creates routing rules for the current NS which direct packets // to the virtual gateway ip on linkToName device interface +// Route 1: 169.254.1.1 dev +// Route 2: default via 169.254.1.1 dev func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { log.Printf("Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0)") virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) @@ -433,6 +448,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { // Helper that creates arp entry for the current NS which maps the virtual // gateway to destMac on a particular interfaceName +// Example: (169.254.1.1) at 12:34:56:78:9a:bc [ether] PERM on func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac @@ -465,6 +481,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { } // Helper function that allows executing a function with one parameter in a VM namespace +// Does not work for process namespaces func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { // Current namespace returnedTo, err := GetCurrentThreadNamespace() From 7c8838b5b21489b10e7164164565d0931bb10f90 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 14 Jul 2022 16:09:00 -0700 Subject: [PATCH 19/86] Namespace netns tests --- network/native_endpointclient_linux_test.go | 72 +++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 7984c5cfcd..459632e347 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -96,6 +96,78 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr: true, wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, + { + name: "Add endpoints get interface fail for getting container veth", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 2), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + }, + { + name: "Add endpoints NetNS Get fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(1, "netns failure"), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + }, + { + name: "Add endpoints NetNS GetFromName fail (with error other than file exists)", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(2, "netns failure"), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + }, + { + name: "Add endpoints NetNS Set fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(3, "netns failure"), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + }, } for _, tt := range tests { From 4e4def5e4f23d48d0ad64a7f31555ea30d17887b Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 14 Jul 2022 16:09:33 -0700 Subject: [PATCH 20/86] Added Native Client to deleteEndpointImpl --- network/endpoint_linux.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index c59ee0a813..82d0e3195a 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -94,6 +94,7 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p if nw.Mode == opModeNative { log.Printf("Mode %s", nw.Mode) log.Printf("Native client") + log.Printf("NSPath: %s, DNS: %s", epInfo.NetNsPath, epInfo.DNS) ethXIfName = fmt.Sprintf("eth0.%d", vlanid) vnetNSName = fmt.Sprintf("az_ns_%d", vlanid) //hostIfName may be a misnomer as this end is in the vnet NS @@ -259,7 +260,27 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. // entering the container netns and hence works both for CNI and CNM. if ep.VlanID != 0 { epInfo := ep.getInfo() - epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) + if nw.Mode == opModeNative { + + log.Printf("Mode %s", nw.Mode) + log.Printf("Native client") + log.Printf("NSPath: %s, DNS: %s", epInfo.NetNsPath, epInfo.DNS) + ethXIfName := fmt.Sprintf("eth0.%d", ep.VlanID) + vnetNSName := fmt.Sprintf("az_ns_%d", ep.VlanID) + //hostIfName may be a misnomer as this end is in the vnet NS + epClient = NewNativeEndpointClient( + nw.extIf.Name, + ethXIfName, + ep.HostIfName, + "", + vnetNSName, + nw.Mode, + ep.VlanID, + nl, + plc) + } else { + epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) + } } else if nw.Mode != opModeTransparent { epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl, plc) } else { From 7e593d939f419ceafbfbdd96b29d55435b7cca5d Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 15 Jul 2022 09:34:07 -0700 Subject: [PATCH 21/86] Deletion of Endpoints Impl and Tests --- network/native_endpointclient_linux.go | 26 ++++++++++++ network/native_endpointclient_linux_test.go | 44 ++++++++++++++++++--- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 5478cb7cb6..02431e850e 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -27,6 +27,7 @@ type NetnsInterface interface { GetFromName(name string) (fileDescriptor uintptr, err error) Set(fileDescriptor uintptr) (err error) NewNamed(name string) (fileDescriptor uintptr, err error) + DeleteNamed(name string) (err error) } type Netns struct{} @@ -48,6 +49,9 @@ func (f *Netns) NewNamed(name string) (uintptr, error) { nsHandle, err := netns.NewNamed(name) return uintptr(nsHandle), err } +func (f *Netns) DeleteNamed(name string) error { + return netns.DeleteNamed(name) +} var ErrorMockNetns = errors.New("mock netns error") @@ -90,6 +94,12 @@ func (f *MockNetns) NewNamed(name string) (uintptr, error) { } return 1, nil } +func (f *MockNetns) DeleteNamed(name string) error { + if f.failMethod == 5 { + return newErrorMockNetns(f.failMessage) + } + return nil +} //End move somewhere else @@ -474,9 +484,25 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { log.Printf("Removing routes") routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { + log.Errorf("Failed to remove routes") return newErrorNativeEndpointClient(err.Error()) } + routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) + + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + log.Printf("There are %d routes remaining: %v", len(routes), routes) + if len(routes) <= 2 { + // Deletes default arp, default routes, ethX + log.Printf("Deleting namespace %s", client.vnetNSName) + delErr := client.netnsClient.DeleteNamed(client.vnetNSName) + if delErr != nil { + log.Errorf("Failed to delete namespace") + return newErrorNativeEndpointClient(delErr.Error()) + } + } return nil } diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 459632e347..8bc03499da 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -263,12 +263,14 @@ func TestNativeDeleteEndpoints(t *testing.T) { plc := platform.NewMockExecClient(false) tests := []struct { - name string - client *NativeEndpointClient - ep *endpoint + name string + client *NativeEndpointClient + ep *endpoint + wantErr bool + wantErrMsg string }{ { - name: "Delete endpoint rules good path", + name: "Delete endpoint good path", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -289,13 +291,45 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, }, }, + wantErr: false, + }, + { + name: "Delete endpoint fail to delete namespace ", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(5, "netns failure"), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + ep: &endpoint{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - tt.client.DeleteEndpointsImpl(tt.ep) + err := tt.client.DeleteEndpointsImpl(tt.ep) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } }) } } From be0191a0dc7e43bdb1f1a1d297d97d30c17a7499 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 15 Jul 2022 16:12:58 -0700 Subject: [PATCH 22/86] Cleaned code (Tests ok) --- network/endpoint_linux.go | 16 +-- network/native_endpointclient_linux.go | 150 ++++++++------------ network/native_endpointclient_linux_test.go | 71 ++++----- 3 files changed, 99 insertions(+), 138 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 82d0e3195a..9d600ea0d5 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -92,9 +92,7 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p if vlanid != 0 { if nw.Mode == opModeNative { - log.Printf("Mode %s", nw.Mode) log.Printf("Native client") - log.Printf("NSPath: %s, DNS: %s", epInfo.NetNsPath, epInfo.DNS) ethXIfName = fmt.Sprintf("eth0.%d", vlanid) vnetNSName = fmt.Sprintf("az_ns_%d", vlanid) //hostIfName may be a misnomer as this end is in the vnet NS @@ -261,23 +259,11 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. if ep.VlanID != 0 { epInfo := ep.getInfo() if nw.Mode == opModeNative { - - log.Printf("Mode %s", nw.Mode) log.Printf("Native client") - log.Printf("NSPath: %s, DNS: %s", epInfo.NetNsPath, epInfo.DNS) ethXIfName := fmt.Sprintf("eth0.%d", ep.VlanID) vnetNSName := fmt.Sprintf("az_ns_%d", ep.VlanID) //hostIfName may be a misnomer as this end is in the vnet NS - epClient = NewNativeEndpointClient( - nw.extIf.Name, - ethXIfName, - ep.HostIfName, - "", - vnetNSName, - nw.Mode, - ep.VlanID, - nl, - plc) + epClient = NewNativeEndpointClient(nw.extIf.Name, ethXIfName, ep.HostIfName, "", vnetNSName, nw.Mode, ep.VlanID, nl, plc) } else { epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) } diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 02431e850e..5e5203660a 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -105,8 +105,8 @@ func (f *MockNetns) DeleteNamed(name string) error { var errorNativeEndpointClient = errors.New("NativeEndpointClient Error") -func newErrorNativeEndpointClient(errStr string) error { - return fmt.Errorf("%w : %s", errorNativeEndpointClient, errStr) +func newErrorNativeEndpointClient(msg string, errStr string) error { + return fmt.Errorf("%w : %s : %s", errorNativeEndpointClient, msg, errStr) } type NativeEndpointClient struct { @@ -142,7 +142,7 @@ func NewNativeEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, ) *NativeEndpointClient { - log.Printf("Create new native client: eth0:%s, ethX:%s, vnet:%s, cont:%s, id:%s", + log.Printf("[native] Create new native client: eth0:%s, ethX:%s, vnet:%s, cont:%s, id:%s", eth0VethName, ethXVethName, vnetVethName, containerVethName, vlanid) client := &NativeEndpointClient{ eth0VethName: eth0VethName, @@ -165,11 +165,12 @@ func NewNativeEndpointClient( // Adds interfaces to the vnet (created if not existing) and vm namespace func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { var err error - err = client.PopulateClient(epInfo) + // VM Namespace + err = client.PopulateVM(epInfo) if err != nil { return err } - + // VNET Namespace err = ExecuteInNS(client.vnetNSName, epInfo, client.PopulateVnet) if err != nil { return err @@ -179,101 +180,94 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } // Called from AddEndpoints, Namespace: VM -func (client *NativeEndpointClient) PopulateClient(epInfo *EndpointInfo) error { - log.Printf("Get VM namespace handle") +func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { vmNS, err := client.netnsClient.Get() if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to get VM NS handle", err.Error()) } - //Mostly for reference + // Inform current namespace returnedTo, err := GetCurrentThreadNamespace() if err != nil { - log.Printf("Unable to get VM namespace: %v", err) - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to get VM NS", err.Error()) } else { - log.Printf("VM Namespace: %s", returnedTo.file.Name()) + log.Printf("[native] VM Namespace: %s", returnedTo.file.Name()) } - log.Printf("Checking if NS exists...") + log.Printf("[native] Checking if NS exists...") var existingErr error vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) - //If the ns does not exist, the below code will trigger to create it + // If the ns does not exist, the below code will trigger to create it if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { - return newErrorNativeEndpointClient(existingErr.Error()) + return newErrorNativeEndpointClient("Error other than vnet NS doesn't exist", existingErr.Error()) } else { - log.Printf("No existing NS detected. Creating the vnet namespace and switching to it") + log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to create vnet NS", err.Error()) } } + } else { + log.Printf("[native] Existing NS detected.") } client.vnetNSFileDescriptor = uintptr(vnetNS) - log.Printf("Set current namespace to VM: %s", vmNS) err = client.netnsClient.Set(vmNS) if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to set current NS to VM", err.Error()) } - log.Printf("Create the host vlan link after getting eth0: %s", client.eth0VethName) + log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) linkAttrs := vishnetlink.NewLinkAttrs() linkAttrs.Name = client.ethXVethName - //Get parent interface index. Index is consistent across libraries. + // Get parent interface index. Index is consistent across libraries. eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) if err != nil { - log.Printf("Failed to get interface: %s", client.eth0VethName) - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to get eth0 interface", err.Error()) } - //Set the peer + // Set the peer linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ LinkAttrs: linkAttrs, VlanId: client.vlanID, } - log.Printf("Add link to VM NS (automatically set to UP)") + log.Printf("[native] Attempting to create eth0.X link in VM NS") // Attempting to create ethX existingErr = vishnetlink.LinkAdd(link) ethXCreated := true if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { - return newErrorNativeEndpointClient(existingErr.Error()) + return newErrorNativeEndpointClient("Error other than eth0.X already exists", existingErr.Error()) } else { - log.Printf("eth0.X already exists") + log.Printf("[native] eth0.X already exists") ethXCreated = false } } if ethXCreated { - log.Printf("Move vlan link (ethX) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) + log.Printf("[native] Move vlan link (eth0.X) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { - log.Printf("Deleting ethX veth in VM NS due to addendpoint failure") if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { log.Errorf("Deleting ethX veth failed on addendpoint failure:%v", delErr) } - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Deleting ethX veth in VM NS due to addendpoint failure", err.Error()) } } - log.Printf("Create veth pair (automatically set to UP)") if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to create veth pair", err.Error()) } - log.Printf("Move vnetVethName into vnet namespace") if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { - log.Printf("Deleting vnet veth due to addendpoint failure") if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { log.Errorf("Deleting vnet veth failed on addendpoint failure:%v", delErr) } - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to move vnetVethName into vnet NS, deleting", err.Error()) } - log.Printf("Check that container veth exists.") containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Container veth does not exist", err.Error()) } client.containerMac = containerIf.HardwareAddr return nil @@ -281,24 +275,15 @@ func (client *NativeEndpointClient) PopulateClient(epInfo *EndpointInfo) error { // Called from AddEndpoints, Namespace: Vnet func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { - - currNS, err := client.netnsClient.Get() - log.Printf("Current NS after switch to vnet: %s", currNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - - log.Printf("Check that (eth0.X) exists") ethXVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("eth0.X doesn't exist", err.Error()) } client.ethXMac = ethXVethIf.HardwareAddr - log.Printf("Check that vnet veth exists") vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("vnet veth doesn't exist", err.Error()) } client.vnetMac = vnetVethIf.HardwareAddr return nil @@ -314,16 +299,14 @@ func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { //Never added any endpoint rules } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { - log.Printf("Moving endpoint to container NS") if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to move endpoint to container NS", err.Error()) } return nil } func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - log.Printf("Setup container interface") if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to setup container interface", err.Error()) } client.containerVethName = epInfo.IfName @@ -332,23 +315,24 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf // Adds routes, arp entries, etc. to the vnet and container namespaces func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + // Container NS err := client.ConfigureContainerInterfacesAndRoutesImpl(epInfo) if err != nil { return err } - //Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes + // Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes err = ExecuteInNS(client.vnetNSName, epInfo, client.ConfigureVnetInterfacesAndRoutesImpl) return err } // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Container func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { - log.Printf("Assign IPs to container veth interface") + if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to assign IPs to container veth interface", err.Error()) } - log.Printf("Remove kernel subnet route automatically added by above") + // kernel subnet route auto added by above call must be removed for _, ipAddr := range epInfo.IPAddresses { _, ipnet, _ := net.ParseCIDR(ipAddr.String()) routeInfo := RouteInfo{ @@ -357,46 +341,38 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(ep Protocol: netlink.RTPROT_KERNEL, } if err := deleteRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { - return newErrorTransparentEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to remove kernel subnet route", err.Error()) } } - log.Printf("Container NS add route for virtual gateway ip") if err := client.AddDefaultRoutes(client.containerVethName); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed Container NS add default routes", err.Error()) } - log.Printf("Container NS add arp entry") if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed Container NS add default arp", err.Error()) } return nil } // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Vnet func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { - log.Printf("Setting vnet loopback state to up") + err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { - log.Printf("Failed to set loopback link state to up") - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to set loopback link state to up", err.Error()) } // Add route specifying which device the pod ip(s) are on routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) - log.Printf("Client data: ethX: %s, vnet: %s", client.ethXVethName, client.vnetVethName) - - log.Printf("Vnet NS add default/gateway routes (Assuming indempotent)") if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed vnet NS add default/gateway routes (indempotent)", err.Error()) } - log.Printf("Vnet NS add default ARP entry (Assuming indempotent)") if err = client.AddDefaultArp(client.ethXVethName, azureMac); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed vnet NS add default ARP entry (idempotent)", err.Error()) } - log.Printf("Adding routes to vnet specific to this container") if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed adding routes to vnet specific to this container", err.Error()) } // Return to ConfigureContainerInterfacesAndRoutes return err @@ -431,7 +407,7 @@ func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []Rou // Route 1: 169.254.1.1 dev // Route 2: default via 169.254.1.1 dev func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { - log.Printf("Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0)") + // Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0) virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ Dst: *virtualGwNet, @@ -442,7 +418,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { return err } - log.Printf("Add default route (ip route add default via 169.254.1.1 dev eth0)") + // Add default route (ip route add default via 169.254.1.1 dev eth0) _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} routeInfo = RouteInfo{ @@ -457,11 +433,10 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { } // Helper that creates arp entry for the current NS which maps the virtual -// gateway to destMac on a particular interfaceName +// gateway (169.254.1.1) to destMac on a particular interfaceName // Example: (169.254.1.1) at 12:34:56:78:9a:bc [ether] PERM on func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) - // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", virtualGwNet.String(), destMac) hardwareAddr, err := net.ParseMAC(destMac) @@ -481,26 +456,23 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { return ExecuteInNS(client.vnetNSName, ep, client.DeleteEndpointsImpl) } func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { - log.Printf("Removing routes") routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - log.Errorf("Failed to remove routes") - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to remove routes", err.Error()) } routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to get route list", err.Error()) } - log.Printf("There are %d routes remaining: %v", len(routes), routes) + log.Printf("[native] There are %d routes remaining: %v", len(routes), routes) if len(routes) <= 2 { - // Deletes default arp, default routes, ethX - log.Printf("Deleting namespace %s", client.vnetNSName) + // Deletes default arp, default routes, ethX; there are two default routes + // so when we have <= 2 routes left, no containers use this namespace + log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) if delErr != nil { - log.Errorf("Failed to delete namespace") - return newErrorNativeEndpointClient(delErr.Error()) + return newErrorNativeEndpointClient("Failed to delete namespace", delErr.Error()) } } return nil @@ -512,7 +484,7 @@ func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { // Current namespace returnedTo, err := GetCurrentThreadNamespace() if err != nil { - log.Printf("[ExecuteInNS] Could not get NS we are in: %v", err) + log.Errorf("[ExecuteInNS] Could not get NS we are in: %v", err) } else { log.Printf("[ExecuteInNS] In NS before switch: %s", returnedTo.file.Name()) } @@ -534,11 +506,11 @@ func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { defer func() { log.Printf("[ExecuteInNS] Exiting vnetns %s.", ns.file.Name()) if err := ns.Exit(); err != nil { - log.Printf("[ExecuteInNS] Could not exit ns, err:%v.", err) + log.Errorf("[ExecuteInNS] Could not exit ns, err:%v.", err) } returnedTo, err := GetCurrentThreadNamespace() if err != nil { - log.Printf("[ExecuteInNS] Could not get NS we returned to: %v", err) + log.Errorf("[ExecuteInNS] Could not get NS we returned to: %v", err) } else { log.Printf("[ExecuteInNS] Returned to NS: %s", returnedTo.file.Name()) } diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 8bc03499da..0278f47fa8 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -25,15 +25,16 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr bool wantErrMsg string }{ + // Populating VM with data and creating interfaces/links { - name: "Add endpoints", + name: "Add endpoints no existing vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: NewMockNetns(2, "no such file or directory"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -43,7 +44,7 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr: false, }, { - name: "Add endpoints netlink fail", + name: "Add endpoints with existing vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -51,17 +52,16 @@ func TestNativeAddEndpoints(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", netnsClient: NewMockNetns(0, ""), - netlink: netlink.NewMockNetlink(true, "netlink fail"), + netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, - epInfo: &EndpointInfo{}, - wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + netlink.ErrorMockNetlink.Error() + " : netlink fail", + epInfo: &EndpointInfo{}, + wantErr: false, }, { - name: "Add endpoints get interface fail for primary interface", + name: "Add endpoints netlink fail", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -69,17 +69,17 @@ func TestNativeAddEndpoints(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", netnsClient: NewMockNetns(0, ""), - netlink: netlink.NewMockNetlink(false, ""), + netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 1), + netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":eth0", + wantErrMsg: "NativeEndpointClient Error : Failed to move vnetVethName into vnet NS, deleting : " + netlink.ErrorMockNetlink.Error() + " : netlink fail", }, { - name: "Add endpoints get interface fail for getting container veth", + name: "Add endpoints get interface fail for primary interface (eth0)", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -90,11 +90,11 @@ func TestNativeAddEndpoints(t *testing.T) { netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 2), + netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + wantErrMsg: "NativeEndpointClient Error : Failed to get eth0 interface : " + netio.ErrMockNetIOFail.Error() + ":eth0", }, { name: "Add endpoints get interface fail for getting container veth", @@ -112,7 +112,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + wantErrMsg: "NativeEndpointClient Error : Container veth does not exist : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, { name: "Add endpoints NetNS Get fail", @@ -130,10 +130,10 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Failed to get VM NS handle : " + ErrorMockNetns.Error() + " : netns failure", }, { - name: "Add endpoints NetNS GetFromName fail (with error other than file exists)", + name: "Add endpoints NetNS GetFromName fail (with error other than file does not exists)", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -148,7 +148,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Error other than vnet NS doesn't exist : " + ErrorMockNetns.Error() + " : netns failure", }, { name: "Add endpoints NetNS Set fail", @@ -166,14 +166,14 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Failed to set current NS to VM : " + ErrorMockNetns.Error() + " : netns failure", }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.PopulateClient(tt.epInfo) + err := tt.client.PopulateVM(tt.epInfo) if tt.wantErr { require.Error(t, err) require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) @@ -190,6 +190,7 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr bool wantErrMsg string }{ + // Populate the client with information from the vnet and set up vnet { name: "Add endpoints second half", client: &NativeEndpointClient{ @@ -208,7 +209,7 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr: false, }, { - name: "Add endpoints netlink fail", + name: "Add endpoints fail check eth0.X exists", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -223,10 +224,10 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":eth0.1", + wantErrMsg: "NativeEndpointClient Error : eth0.X doesn't exist : " + netio.ErrMockNetIOFail.Error() + ":eth0.1", }, { - name: "Add endpoints netlink fail", + name: "Add endpoints fail check vnet veth exists", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -241,7 +242,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":A1veth0", + wantErrMsg: "NativeEndpointClient Error : vnet veth doesn't exist : " + netio.ErrMockNetIOFail.Error() + ":A1veth0", }, } @@ -293,8 +294,9 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, wantErr: false, }, + // You must have <= 2 ip routes on your machine for this to pass { - name: "Delete endpoint fail to delete namespace ", + name: "Delete endpoint fail to delete namespace", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -316,7 +318,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Failed to delete namespace : " + ErrorMockNetns.Error() + " : netns failure", }, } @@ -348,7 +350,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErrMsg string }{ { - name: "Configure Interface and routes good path", + name: "Configure interface and routes good path for container", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -373,7 +375,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErr: false, }, { - name: "Configure Interface and routes multiple IPs", + name: "Configure interface and routes multiple IPs", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -406,7 +408,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErr: false, }, { - name: "Configure Interface and routes assign ip fail", + name: "Configure interface and routes assign ip fail", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -432,7 +434,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErrMsg: "netlink fail", }, { - name: "Configure Interface and routes container final route (2nd default route) fail", + name: "Configure interface and routes container 2nd default route added fail", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -455,7 +457,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + wantErrMsg: "NativeEndpointClient Error : Failed Container NS add default routes : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, } @@ -479,7 +481,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErrMsg string }{ { - name: "Configure Interface and routes good path second half", + name: "Configure interface and routes good path for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -504,7 +506,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErr: false, }, { - name: "Configure Interface and routes final routes for vnet", + // fail route that tells which device container ip is on for vnet + name: "Configure interface and routes fail final routes for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -527,7 +530,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", + wantErrMsg: "NativeEndpointClient Error : Failed adding routes to vnet specific to this container : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", }, } From 1f40dd876db0c3f42d783cd272826d98c862ba58 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 18 Jul 2022 10:15:00 -0700 Subject: [PATCH 23/86] Moved mock/netns to package (Tests ok) --- netns/mocknetns.go | 55 ++++++++++++ netns/netns.go | 28 ++++++ netns/netnsinterface.go | 9 ++ network/native_endpointclient_linux.go | 97 ++------------------- network/native_endpointclient_linux_test.go | 47 +++++----- 5 files changed, 124 insertions(+), 112 deletions(-) create mode 100644 netns/mocknetns.go create mode 100644 netns/netns.go create mode 100644 netns/netnsinterface.go diff --git a/netns/mocknetns.go b/netns/mocknetns.go new file mode 100644 index 0000000000..fed1b4b2c4 --- /dev/null +++ b/netns/mocknetns.go @@ -0,0 +1,55 @@ +package netns + +import ( + "errors" + "fmt" +) + +var ErrorMockNetns = errors.New("mock netns error") + +func newErrorMockNetns(errStr string) error { + return fmt.Errorf("%w : %s", ErrorMockNetns, errStr) +} + +type MockNetns struct { + failMethod int + failMessage string +} + +func NewMockNetns(failMethod int, failMessage string) *MockNetns { + return &MockNetns{ + failMethod: failMethod, + failMessage: failMessage, + } +} + +func (f *MockNetns) Get() (uintptr, error) { + if f.failMethod == 1 { + return 0, newErrorMockNetns(f.failMessage) + } + return 1, nil +} +func (f *MockNetns) GetFromName(name string) (uintptr, error) { + if f.failMethod == 2 { + return 0, newErrorMockNetns(f.failMessage) + } + return 1, nil +} +func (f *MockNetns) Set(handle uintptr) error { + if f.failMethod == 3 { + return newErrorMockNetns(f.failMessage) + } + return nil +} +func (f *MockNetns) NewNamed(name string) (uintptr, error) { + if f.failMethod == 4 { + return 0, newErrorMockNetns(f.failMessage) + } + return 1, nil +} +func (f *MockNetns) DeleteNamed(name string) error { + if f.failMethod == 5 { + return newErrorMockNetns(f.failMessage) + } + return nil +} diff --git a/netns/netns.go b/netns/netns.go new file mode 100644 index 0000000000..61a5655bca --- /dev/null +++ b/netns/netns.go @@ -0,0 +1,28 @@ +package netns + +import "github.com/vishvananda/netns" + +type Netns struct{} + +func NewNetns() *Netns { + return &Netns{} +} + +func (f *Netns) Get() (uintptr, error) { + nsHandle, err := netns.Get() + return uintptr(nsHandle), err +} +func (f *Netns) GetFromName(name string) (uintptr, error) { + nsHandle, err := netns.GetFromName(name) + return uintptr(nsHandle), err +} +func (f *Netns) Set(fileDescriptor uintptr) error { + return netns.Set(netns.NsHandle(fileDescriptor)) +} +func (f *Netns) NewNamed(name string) (uintptr, error) { + nsHandle, err := netns.NewNamed(name) + return uintptr(nsHandle), err +} +func (f *Netns) DeleteNamed(name string) error { + return netns.DeleteNamed(name) +} diff --git a/netns/netnsinterface.go b/netns/netnsinterface.go new file mode 100644 index 0000000000..7960d5fd21 --- /dev/null +++ b/netns/netnsinterface.go @@ -0,0 +1,9 @@ +package netns + +type NetnsInterface interface { + Get() (fileDescriptor uintptr, err error) + GetFromName(name string) (fileDescriptor uintptr, err error) + Set(fileDescriptor uintptr) (err error) + NewNamed(name string) (fileDescriptor uintptr, err error) + DeleteNamed(name string) (err error) +} diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 5e5203660a..dfb7772ccd 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -9,100 +9,19 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" vishnetlink "github.com/vishvananda/netlink" - "github.com/vishvananda/netns" ) const ( - azureMac = "12:34:56:78:9a:bc" - loopbackIf = "lo" + azureMac = "12:34:56:78:9a:bc" // Packets leaving the VM should have this MAC + loopbackIf = "lo" // The name of the loopback interface + numDefaultRoutes = 2 // VNET NS, when no containers use it, has this many routes ) -//Move somewhere else -type NetnsInterface interface { - Get() (fileDescriptor uintptr, err error) - GetFromName(name string) (fileDescriptor uintptr, err error) - Set(fileDescriptor uintptr) (err error) - NewNamed(name string) (fileDescriptor uintptr, err error) - DeleteNamed(name string) (err error) -} -type Netns struct{} - -func NewNetns() *Netns { - return &Netns{} -} -func (f *Netns) Get() (uintptr, error) { - nsHandle, err := netns.Get() - return uintptr(nsHandle), err -} -func (f *Netns) GetFromName(name string) (uintptr, error) { - nsHandle, err := netns.GetFromName(name) - return uintptr(nsHandle), err -} -func (f *Netns) Set(fileDescriptor uintptr) error { - return netns.Set(netns.NsHandle(fileDescriptor)) -} -func (f *Netns) NewNamed(name string) (uintptr, error) { - nsHandle, err := netns.NewNamed(name) - return uintptr(nsHandle), err -} -func (f *Netns) DeleteNamed(name string) error { - return netns.DeleteNamed(name) -} - -var ErrorMockNetns = errors.New("mock netns error") - -func newErrorMockNetns(errStr string) error { - return fmt.Errorf("%w : %s", ErrorMockNetns, errStr) -} - -type MockNetns struct { - failMethod int - failMessage string -} - -func NewMockNetns(failMethod int, failMessage string) *MockNetns { - return &MockNetns{ - failMethod: failMethod, - failMessage: failMessage, - } -} -func (f *MockNetns) Get() (uintptr, error) { - if f.failMethod == 1 { - return 0, newErrorMockNetns(f.failMessage) - } - return 1, nil -} -func (f *MockNetns) GetFromName(name string) (uintptr, error) { - if f.failMethod == 2 { - return 0, newErrorMockNetns(f.failMessage) - } - return 1, nil -} -func (f *MockNetns) Set(handle uintptr) error { - if f.failMethod == 3 { - return newErrorMockNetns(f.failMessage) - } - return nil -} -func (f *MockNetns) NewNamed(name string) (uintptr, error) { - if f.failMethod == 4 { - return 0, newErrorMockNetns(f.failMessage) - } - return 1, nil -} -func (f *MockNetns) DeleteNamed(name string) error { - if f.failMethod == 5 { - return newErrorMockNetns(f.failMessage) - } - return nil -} - -//End move somewhere else - var errorNativeEndpointClient = errors.New("NativeEndpointClient Error") func newErrorNativeEndpointClient(msg string, errStr string) error { @@ -124,7 +43,7 @@ type NativeEndpointClient struct { mode string vlanID int - netnsClient NetnsInterface + netnsClient netns.NetnsInterface netlink netlink.NetlinkInterface netioshim netio.NetIOInterface plClient platform.ExecClient @@ -152,7 +71,7 @@ func NewNativeEndpointClient( vnetNSName: vnetNSName, mode: mode, vlanID: vlanid, - netnsClient: NewNetns(), + netnsClient: netns.NewNetns(), netlink: nl, netioshim: &netio.NetIO{}, plClient: plc, @@ -466,9 +385,9 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { return newErrorNativeEndpointClient("Failed to get route list", err.Error()) } log.Printf("[native] There are %d routes remaining: %v", len(routes), routes) - if len(routes) <= 2 { + if len(routes) <= numDefaultRoutes { // Deletes default arp, default routes, ethX; there are two default routes - // so when we have <= 2 routes left, no containers use this namespace + // so when we have <= numDefaultRoutes routes left, no containers use this namespace log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) if delErr != nil { diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 0278f47fa8..99788dba90 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" "github.com/stretchr/testify/require" @@ -34,7 +35,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(2, "no such file or directory"), + netnsClient: netns.NewMockNetns(2, "no such file or directory"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -51,7 +52,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -68,7 +69,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -86,7 +87,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -104,7 +105,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -122,7 +123,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(1, "netns failure"), + netnsClient: netns.NewMockNetns(1, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -130,7 +131,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to get VM NS handle : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Failed to get VM NS handle : " + netns.ErrorMockNetns.Error() + " : netns failure", }, { name: "Add endpoints NetNS GetFromName fail (with error other than file does not exists)", @@ -140,7 +141,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(2, "netns failure"), + netnsClient: netns.NewMockNetns(2, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -148,7 +149,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Error other than vnet NS doesn't exist : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Error other than vnet NS doesn't exist : " + netns.ErrorMockNetns.Error() + " : netns failure", }, { name: "Add endpoints NetNS Set fail", @@ -158,7 +159,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(3, "netns failure"), + netnsClient: netns.NewMockNetns(3, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -166,7 +167,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to set current NS to VM : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Failed to set current NS to VM : " + netns.ErrorMockNetns.Error() + " : netns failure", }, } @@ -199,7 +200,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -216,7 +217,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -234,7 +235,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -278,7 +279,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -303,7 +304,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(5, "netns failure"), + netnsClient: netns.NewMockNetns(5, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -318,7 +319,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to delete namespace : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Failed to delete namespace : " + netns.ErrorMockNetns.Error() + " : netns failure", }, } @@ -358,7 +359,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -383,7 +384,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -416,7 +417,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -442,7 +443,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -489,7 +490,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -515,7 +516,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), From 05524935c44ecdea2423840a4496107a6ac8a353 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 18 Jul 2022 15:51:23 -0700 Subject: [PATCH 24/86] Fixing Netns (wip) Moved netnsinterface to consumer package (network). Removed "Netns" from "NewNetns" and "NewMockNetns" as it is unambiguous. Changed uintptr to int and casted the int to uintptr when needed later. --- netns/mocknetns.go | 2 +- netns/netns.go | 16 ++++++++-------- netns/netnsinterface.go | 9 --------- network/native_endpointclient_linux.go | 8 ++++---- network/netnsinterface.go | 9 +++++++++ 5 files changed, 22 insertions(+), 22 deletions(-) delete mode 100644 netns/netnsinterface.go create mode 100644 network/netnsinterface.go diff --git a/netns/mocknetns.go b/netns/mocknetns.go index fed1b4b2c4..2eb9f197a6 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -16,7 +16,7 @@ type MockNetns struct { failMessage string } -func NewMockNetns(failMethod int, failMessage string) *MockNetns { +func NewMock(failMethod int, failMessage string) *MockNetns { return &MockNetns{ failMethod: failMethod, failMessage: failMessage, diff --git a/netns/netns.go b/netns/netns.go index 61a5655bca..1389db124b 100644 --- a/netns/netns.go +++ b/netns/netns.go @@ -4,24 +4,24 @@ import "github.com/vishvananda/netns" type Netns struct{} -func NewNetns() *Netns { +func New() *Netns { return &Netns{} } -func (f *Netns) Get() (uintptr, error) { +func (f *Netns) Get() (int, error) { nsHandle, err := netns.Get() - return uintptr(nsHandle), err + return int(nsHandle), err } -func (f *Netns) GetFromName(name string) (uintptr, error) { +func (f *Netns) GetFromName(name string) (int, error) { nsHandle, err := netns.GetFromName(name) - return uintptr(nsHandle), err + return int(nsHandle), err } -func (f *Netns) Set(fileDescriptor uintptr) error { +func (f *Netns) Set(fileDescriptor int) error { return netns.Set(netns.NsHandle(fileDescriptor)) } -func (f *Netns) NewNamed(name string) (uintptr, error) { +func (f *Netns) NewNamed(name string) (int, error) { nsHandle, err := netns.NewNamed(name) - return uintptr(nsHandle), err + return int(nsHandle), err } func (f *Netns) DeleteNamed(name string) error { return netns.DeleteNamed(name) diff --git a/netns/netnsinterface.go b/netns/netnsinterface.go deleted file mode 100644 index 7960d5fd21..0000000000 --- a/netns/netnsinterface.go +++ /dev/null @@ -1,9 +0,0 @@ -package netns - -type NetnsInterface interface { - Get() (fileDescriptor uintptr, err error) - GetFromName(name string) (fileDescriptor uintptr, err error) - Set(fileDescriptor uintptr) (err error) - NewNamed(name string) (fileDescriptor uintptr, err error) - DeleteNamed(name string) (err error) -} diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index dfb7772ccd..80ff607ddc 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -39,11 +39,11 @@ type NativeEndpointClient struct { ethXMac net.HardwareAddr vnetNSName string - vnetNSFileDescriptor uintptr + vnetNSFileDescriptor int mode string vlanID int - netnsClient netns.NetnsInterface + netnsClient NetnsInterface netlink netlink.NetlinkInterface netioshim netio.NetIOInterface plClient platform.ExecClient @@ -71,7 +71,7 @@ func NewNativeEndpointClient( vnetNSName: vnetNSName, mode: mode, vlanID: vlanid, - netnsClient: netns.NewNetns(), + netnsClient: netns.New(), netlink: nl, netioshim: &netio.NetIO{}, plClient: plc, @@ -130,7 +130,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { } else { log.Printf("[native] Existing NS detected.") } - client.vnetNSFileDescriptor = uintptr(vnetNS) + client.vnetNSFileDescriptor = vnetNS err = client.netnsClient.Set(vmNS) if err != nil { diff --git a/network/netnsinterface.go b/network/netnsinterface.go new file mode 100644 index 0000000000..8f6311c386 --- /dev/null +++ b/network/netnsinterface.go @@ -0,0 +1,9 @@ +package network + +type NetnsInterface interface { + Get() (fileDescriptor int, err error) + GetFromName(name string) (fileDescriptor int, err error) + Set(fileDescriptor int) (err error) + NewNamed(name string) (fileDescriptor int, err error) + DeleteNamed(name string) (err error) +} From 2c2aac5b1e2c2ae6fa9d5cd55a6b2fb49a27daf1 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 18 Jul 2022 16:54:38 -0700 Subject: [PATCH 25/86] Using errors.Wrap for error context (wip) --- netns/mocknetns.go | 5 +-- network/native_endpointclient_linux.go | 60 ++++++++++++-------------- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index 2eb9f197a6..08e69d65f3 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -1,14 +1,13 @@ package netns import ( - "errors" - "fmt" + "github.com/pkg/errors" ) var ErrorMockNetns = errors.New("mock netns error") func newErrorMockNetns(errStr string) error { - return fmt.Errorf("%w : %s", ErrorMockNetns, errStr) + return errors.Wrap(ErrorMockNetns, errStr) } type MockNetns struct { diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 80ff607ddc..153a117e85 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -1,7 +1,6 @@ package network import ( - "errors" "fmt" "net" "strings" @@ -12,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" + "github.com/pkg/errors" vishnetlink "github.com/vishvananda/netlink" ) @@ -22,12 +22,6 @@ const ( numDefaultRoutes = 2 // VNET NS, when no containers use it, has this many routes ) -var errorNativeEndpointClient = errors.New("NativeEndpointClient Error") - -func newErrorNativeEndpointClient(msg string, errStr string) error { - return fmt.Errorf("%w : %s : %s", errorNativeEndpointClient, msg, errStr) -} - type NativeEndpointClient struct { eth0VethName string //So like eth0 ethXVethName string //So like eth0.X @@ -102,12 +96,12 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { vmNS, err := client.netnsClient.Get() if err != nil { - return newErrorNativeEndpointClient("Failed to get VM NS handle", err.Error()) + return errors.Wrap(err, "Failed to get VM NS handle") } // Inform current namespace returnedTo, err := GetCurrentThreadNamespace() if err != nil { - return newErrorNativeEndpointClient("Failed to get VM NS", err.Error()) + return errors.Wrap(err, "Failed to get VM NS") } else { log.Printf("[native] VM Namespace: %s", returnedTo.file.Name()) } @@ -118,12 +112,12 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // If the ns does not exist, the below code will trigger to create it if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { - return newErrorNativeEndpointClient("Error other than vnet NS doesn't exist", existingErr.Error()) + return errors.Wrap(existingErr, "Error other than vnet NS doesn't exist") } else { log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) if err != nil { - return newErrorNativeEndpointClient("Failed to create vnet NS", err.Error()) + return errors.Wrap(err, "Failed to create vnet NS") } } @@ -134,7 +128,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { err = client.netnsClient.Set(vmNS) if err != nil { - return newErrorNativeEndpointClient("Failed to set current NS to VM", err.Error()) + return errors.Wrap(err, "Failed to set current NS to VM") } log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) @@ -143,7 +137,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Get parent interface index. Index is consistent across libraries. eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) if err != nil { - return newErrorNativeEndpointClient("Failed to get eth0 interface", err.Error()) + return errors.Wrap(err, "Failed to get eth0 interface") } // Set the peer linkAttrs.ParentIndex = eth0.Index @@ -157,7 +151,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { ethXCreated := true if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { - return newErrorNativeEndpointClient("Error other than eth0.X already exists", existingErr.Error()) + return errors.Wrap(err, "Error other than eth0.X already exists") } else { log.Printf("[native] eth0.X already exists") ethXCreated = false @@ -169,24 +163,24 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { log.Errorf("Deleting ethX veth failed on addendpoint failure:%v", delErr) } - return newErrorNativeEndpointClient("Deleting ethX veth in VM NS due to addendpoint failure", err.Error()) + return errors.Wrap(err, "Deleting ethX veth in VM NS due to addendpoint failure") } } if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { - return newErrorNativeEndpointClient("Failed to create veth pair", err.Error()) + return errors.Wrap(err, "Failed to create veth pair") } if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { log.Errorf("Deleting vnet veth failed on addendpoint failure:%v", delErr) } - return newErrorNativeEndpointClient("Failed to move vnetVethName into vnet NS, deleting", err.Error()) + return errors.Wrap(err, "Failed to move vnetVethName into vnet NS, deleting") } containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) if err != nil { - return newErrorNativeEndpointClient("Container veth does not exist", err.Error()) + return errors.Wrap(err, "Container veth does not exist") } client.containerMac = containerIf.HardwareAddr return nil @@ -196,13 +190,13 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { ethXVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) if err != nil { - return newErrorNativeEndpointClient("eth0.X doesn't exist", err.Error()) + return errors.Wrap(err, "eth0.X doesn't exist") } client.ethXMac = ethXVethIf.HardwareAddr vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) if err != nil { - return newErrorNativeEndpointClient("vnet veth doesn't exist", err.Error()) + return errors.Wrap(err, "vnet veth doesn't exist") } client.vnetMac = vnetVethIf.HardwareAddr return nil @@ -219,13 +213,13 @@ func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { - return newErrorNativeEndpointClient("Failed to move endpoint to container NS", err.Error()) + return errors.Wrap(err, "Failed to move endpoint to container NS") } return nil } func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { - return newErrorNativeEndpointClient("Failed to setup container interface", err.Error()) + return errors.Wrap(err, "Failed to setup container interface") } client.containerVethName = epInfo.IfName @@ -249,7 +243,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { - return newErrorNativeEndpointClient("Failed to assign IPs to container veth interface", err.Error()) + return errors.Wrap(err, "Failed to assign IPs to container veth interface") } // kernel subnet route auto added by above call must be removed for _, ipAddr := range epInfo.IPAddresses { @@ -260,15 +254,15 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(ep Protocol: netlink.RTPROT_KERNEL, } if err := deleteRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { - return newErrorNativeEndpointClient("Failed to remove kernel subnet route", err.Error()) + return errors.Wrap(err, "Failed to remove kernel subnet route") } } if err := client.AddDefaultRoutes(client.containerVethName); err != nil { - return newErrorNativeEndpointClient("Failed Container NS add default routes", err.Error()) + return errors.Wrap(err, "Failed Container NS add default routes") } if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { - return newErrorNativeEndpointClient("Failed Container NS add default arp", err.Error()) + return errors.Wrap(err, "Failed Container NS add default arp") } return nil } @@ -278,20 +272,20 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { - return newErrorNativeEndpointClient("Failed to set loopback link state to up", err.Error()) + return errors.Wrap(err, "Failed to set loopback link state to up") } // Add route specifying which device the pod ip(s) are on routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { - return newErrorNativeEndpointClient("Failed vnet NS add default/gateway routes (indempotent)", err.Error()) + return errors.Wrap(err, "Failed vnet NS add default/gateway routes (indempotent)") } if err = client.AddDefaultArp(client.ethXVethName, azureMac); err != nil { - return newErrorNativeEndpointClient("Failed vnet NS add default ARP entry (idempotent)", err.Error()) + return errors.Wrap(err, "Failed vnet NS add default ARP entry (idempotent)") } if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return newErrorNativeEndpointClient("Failed adding routes to vnet specific to this container", err.Error()) + return errors.Wrap(err, "Failed adding routes to vnet specific to this container") } // Return to ConfigureContainerInterfacesAndRoutes return err @@ -377,12 +371,12 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return newErrorNativeEndpointClient("Failed to remove routes", err.Error()) + return errors.Wrap(err, "Failed to remove routes") } routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) if err != nil { - return newErrorNativeEndpointClient("Failed to get route list", err.Error()) + return errors.Wrap(err, ("Failed to get route list")) } log.Printf("[native] There are %d routes remaining: %v", len(routes), routes) if len(routes) <= numDefaultRoutes { @@ -391,7 +385,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) if delErr != nil { - return newErrorNativeEndpointClient("Failed to delete namespace", delErr.Error()) + return errors.Wrap(err, "Failed to delete namespace") } } return nil From 248a90c96d3b3e18905804821e8648518be17c9f Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 18 Jul 2022 17:00:02 -0700 Subject: [PATCH 26/86] Removed sentence case (wip) --- network/native_endpointclient_linux.go | 52 +++++++++++++------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 153a117e85..e2cbee70ef 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -96,12 +96,12 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { vmNS, err := client.netnsClient.Get() if err != nil { - return errors.Wrap(err, "Failed to get VM NS handle") + return errors.Wrap(err, "failed to get vm ns handle") } // Inform current namespace returnedTo, err := GetCurrentThreadNamespace() if err != nil { - return errors.Wrap(err, "Failed to get VM NS") + return errors.Wrap(err, "failed to get vm ns") } else { log.Printf("[native] VM Namespace: %s", returnedTo.file.Name()) } @@ -112,12 +112,12 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // If the ns does not exist, the below code will trigger to create it if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { - return errors.Wrap(existingErr, "Error other than vnet NS doesn't exist") + return errors.Wrap(existingErr, "error other than vnet ns doesn't exist") } else { log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) if err != nil { - return errors.Wrap(err, "Failed to create vnet NS") + return errors.Wrap(err, "failed to create vnet ns") } } @@ -128,7 +128,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { err = client.netnsClient.Set(vmNS) if err != nil { - return errors.Wrap(err, "Failed to set current NS to VM") + return errors.Wrap(err, "failed to set current ns to vm") } log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) @@ -137,7 +137,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Get parent interface index. Index is consistent across libraries. eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) if err != nil { - return errors.Wrap(err, "Failed to get eth0 interface") + return errors.Wrap(err, "failed to get eth0 interface") } // Set the peer linkAttrs.ParentIndex = eth0.Index @@ -151,7 +151,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { ethXCreated := true if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { - return errors.Wrap(err, "Error other than eth0.X already exists") + return errors.Wrap(err, "error other than eth0.x already exists") } else { log.Printf("[native] eth0.X already exists") ethXCreated = false @@ -161,26 +161,26 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { log.Printf("[native] Move vlan link (eth0.X) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { - log.Errorf("Deleting ethX veth failed on addendpoint failure:%v", delErr) + log.Errorf("deleting ethx veth failed on addendpoint failure:%v", delErr) } - return errors.Wrap(err, "Deleting ethX veth in VM NS due to addendpoint failure") + return errors.Wrap(err, "deleting ethx veth in vm ns due to addendpoint failure") } } if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { - return errors.Wrap(err, "Failed to create veth pair") + return errors.Wrap(err, "failed to create veth pair") } if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { log.Errorf("Deleting vnet veth failed on addendpoint failure:%v", delErr) } - return errors.Wrap(err, "Failed to move vnetVethName into vnet NS, deleting") + return errors.Wrap(err, "failed to move vnetVethName into vnet ns, deleting") } containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) if err != nil { - return errors.Wrap(err, "Container veth does not exist") + return errors.Wrap(err, "container veth does not exist") } client.containerMac = containerIf.HardwareAddr return nil @@ -190,7 +190,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { ethXVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) if err != nil { - return errors.Wrap(err, "eth0.X doesn't exist") + return errors.Wrap(err, "eth0.x doesn't exist") } client.ethXMac = ethXVethIf.HardwareAddr @@ -213,13 +213,13 @@ func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { - return errors.Wrap(err, "Failed to move endpoint to container NS") + return errors.Wrap(err, "failed to move endpoint to container ns") } return nil } func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { - return errors.Wrap(err, "Failed to setup container interface") + return errors.Wrap(err, "failed to setup container interface") } client.containerVethName = epInfo.IfName @@ -243,7 +243,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { - return errors.Wrap(err, "Failed to assign IPs to container veth interface") + return errors.Wrap(err, "failed to assign ips to container veth interface") } // kernel subnet route auto added by above call must be removed for _, ipAddr := range epInfo.IPAddresses { @@ -254,15 +254,15 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(ep Protocol: netlink.RTPROT_KERNEL, } if err := deleteRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { - return errors.Wrap(err, "Failed to remove kernel subnet route") + return errors.Wrap(err, "failed to remove kernel subnet route") } } if err := client.AddDefaultRoutes(client.containerVethName); err != nil { - return errors.Wrap(err, "Failed Container NS add default routes") + return errors.Wrap(err, "failed Container ns add default routes") } if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { - return errors.Wrap(err, "Failed Container NS add default arp") + return errors.Wrap(err, "failed Container ns add default arp") } return nil } @@ -272,20 +272,20 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { - return errors.Wrap(err, "Failed to set loopback link state to up") + return errors.Wrap(err, "failed to set loopback link state to up") } // Add route specifying which device the pod ip(s) are on routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { - return errors.Wrap(err, "Failed vnet NS add default/gateway routes (indempotent)") + return errors.Wrap(err, "failed vnet ns add default/gateway routes (indempotent)") } if err = client.AddDefaultArp(client.ethXVethName, azureMac); err != nil { - return errors.Wrap(err, "Failed vnet NS add default ARP entry (idempotent)") + return errors.Wrap(err, "failed vnet ns add default arp entry (idempotent)") } if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return errors.Wrap(err, "Failed adding routes to vnet specific to this container") + return errors.Wrap(err, "failed adding routes to vnet specific to this container") } // Return to ConfigureContainerInterfacesAndRoutes return err @@ -371,12 +371,12 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return errors.Wrap(err, "Failed to remove routes") + return errors.Wrap(err, "failed to remove routes") } routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) if err != nil { - return errors.Wrap(err, ("Failed to get route list")) + return errors.Wrap(err, ("failed to get route list")) } log.Printf("[native] There are %d routes remaining: %v", len(routes), routes) if len(routes) <= numDefaultRoutes { @@ -385,7 +385,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) if delErr != nil { - return errors.Wrap(err, "Failed to delete namespace") + return errors.Wrap(err, "failed to delete namespace") } } return nil From 3e8e7191f6b0e2ce81d0f3f565327c3a5c533700 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:11:51 -0700 Subject: [PATCH 27/86] Removing variable predeclaration --- network/native_endpointclient_linux.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index e2cbee70ef..70766a40c5 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -77,9 +77,8 @@ func NewNativeEndpointClient( // Adds interfaces to the vnet (created if not existing) and vm namespace func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { - var err error // VM Namespace - err = client.PopulateVM(epInfo) + err := client.PopulateVM(epInfo) if err != nil { return err } @@ -107,9 +106,9 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { } log.Printf("[native] Checking if NS exists...") - var existingErr error vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) // If the ns does not exist, the below code will trigger to create it + if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { return errors.Wrap(existingErr, "error other than vnet ns doesn't exist") From a3b6277b7cb40bc1d2a0004e827d2543d66520b8 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:21:12 -0700 Subject: [PATCH 28/86] Removed NewNativeEndpointClient Directly instantiating struct because nothing special happens in NewNativeEndpointClient --- network/endpoint_linux.go | 44 ++++++++++++++++++-------- network/native_endpointclient_linux.go | 32 ------------------- 2 files changed, 31 insertions(+), 45 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 9d600ea0d5..d035c4990c 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" "github.com/Azure/azure-container-networking/platform" @@ -95,17 +96,21 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p log.Printf("Native client") ethXIfName = fmt.Sprintf("eth0.%d", vlanid) vnetNSName = fmt.Sprintf("az_ns_%d", vlanid) - //hostIfName may be a misnomer as this end is in the vnet NS - epClient = NewNativeEndpointClient( - nw.extIf.Name, - ethXIfName, - hostIfName, - contIfName, - vnetNSName, - nw.Mode, - vlanid, - nl, - plc) + + epClient = &NativeEndpointClient{ + eth0VethName: nw.extIf.Name, + ethXVethName: ethXIfName, + vnetVethName: hostIfName, + containerVethName: contIfName, + vnetNSName: vnetNSName, + mode: nw.Mode, + vlanID: vlanid, + netnsClient: netns.New(), + netlink: nl, + netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + } } else { log.Printf("OVS client") if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { @@ -262,8 +267,21 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. log.Printf("Native client") ethXIfName := fmt.Sprintf("eth0.%d", ep.VlanID) vnetNSName := fmt.Sprintf("az_ns_%d", ep.VlanID) - //hostIfName may be a misnomer as this end is in the vnet NS - epClient = NewNativeEndpointClient(nw.extIf.Name, ethXIfName, ep.HostIfName, "", vnetNSName, nw.Mode, ep.VlanID, nl, plc) + + epClient = &NativeEndpointClient{ + eth0VethName: nw.extIf.Name, + ethXVethName: ethXIfName, + vnetVethName: ep.HostIfName, + containerVethName: "", + vnetNSName: vnetNSName, + mode: nw.Mode, + vlanID: ep.VlanID, + netnsClient: netns.New(), + netlink: nl, + netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + } } else { epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) } diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 70766a40c5..dd4b479d37 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -8,7 +8,6 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" "github.com/pkg/errors" @@ -44,37 +43,6 @@ type NativeEndpointClient struct { netUtilsClient networkutils.NetworkUtils } -func NewNativeEndpointClient( - eth0VethName string, - ethXVethName string, - vnetVethName string, - containerVethName string, - vnetNSName string, - mode string, - vlanid int, - nl netlink.NetlinkInterface, - plc platform.ExecClient, -) *NativeEndpointClient { - log.Printf("[native] Create new native client: eth0:%s, ethX:%s, vnet:%s, cont:%s, id:%s", - eth0VethName, ethXVethName, vnetVethName, containerVethName, vlanid) - client := &NativeEndpointClient{ - eth0VethName: eth0VethName, - ethXVethName: ethXVethName, - vnetVethName: vnetVethName, - containerVethName: containerVethName, - vnetNSName: vnetNSName, - mode: mode, - vlanID: vlanid, - netnsClient: netns.New(), - netlink: nl, - netioshim: &netio.NetIO{}, - plClient: plc, - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - } - - return client -} - // Adds interfaces to the vnet (created if not existing) and vm namespace func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { // VM Namespace From 536a022978ca22d00c37339227e7f0be84b7ae56 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:30:35 -0700 Subject: [PATCH 29/86] Removed generics from ExecuteInNS --- network/native_endpointclient_linux.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index dd4b479d37..7b0cd6d4e7 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -51,12 +51,9 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return err } // VNET Namespace - err = ExecuteInNS(client.vnetNSName, epInfo, client.PopulateVnet) - if err != nil { - return err - } - - return nil + return ExecuteInNS(client.vnetNSName, func() error { + return client.PopulateVnet(epInfo) + }) } // Called from AddEndpoints, Namespace: VM @@ -202,8 +199,9 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo } // Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes - err = ExecuteInNS(client.vnetNSName, epInfo, client.ConfigureVnetInterfacesAndRoutesImpl) - return err + return ExecuteInNS(client.vnetNSName, func() error { + return client.ConfigureVnetInterfacesAndRoutesImpl(epInfo) + }) } // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Container @@ -333,7 +331,9 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac return nil } func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { - return ExecuteInNS(client.vnetNSName, ep, client.DeleteEndpointsImpl) + return ExecuteInNS(client.vnetNSName, func() error { + return client.DeleteEndpointsImpl(ep) + }) } func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) @@ -360,7 +360,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { // Helper function that allows executing a function with one parameter in a VM namespace // Does not work for process namespaces -func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { +func ExecuteInNS(nsName string, f func() error) error { // Current namespace returnedTo, err := GetCurrentThreadNamespace() if err != nil { @@ -395,5 +395,5 @@ func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { log.Printf("[ExecuteInNS] Returned to NS: %s", returnedTo.file.Name()) } }() - return f(param) + return f() } From 9193600fef0809bd0180929b036f5da8cf8a2846 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:43:25 -0700 Subject: [PATCH 30/86] Removed uintptr from mocknetns, tests compile Forgot to remove uintptr from mocknetns --- netns/mocknetns.go | 8 ++--- network/native_endpointclient_linux_test.go | 38 ++++++++++----------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index 08e69d65f3..4026de8ab0 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -22,25 +22,25 @@ func NewMock(failMethod int, failMessage string) *MockNetns { } } -func (f *MockNetns) Get() (uintptr, error) { +func (f *MockNetns) Get() (int, error) { if f.failMethod == 1 { return 0, newErrorMockNetns(f.failMessage) } return 1, nil } -func (f *MockNetns) GetFromName(name string) (uintptr, error) { +func (f *MockNetns) GetFromName(name string) (int, error) { if f.failMethod == 2 { return 0, newErrorMockNetns(f.failMessage) } return 1, nil } -func (f *MockNetns) Set(handle uintptr) error { +func (f *MockNetns) Set(handle int) error { if f.failMethod == 3 { return newErrorMockNetns(f.failMessage) } return nil } -func (f *MockNetns) NewNamed(name string) (uintptr, error) { +func (f *MockNetns) NewNamed(name string) (int, error) { if f.failMethod == 4 { return 0, newErrorMockNetns(f.failMessage) } diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 99788dba90..3a2f9d71b1 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -35,7 +35,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(2, "no such file or directory"), + netnsClient: netns.NewMock(2, "no such file or directory"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -52,7 +52,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -69,7 +69,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -87,7 +87,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -105,7 +105,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -123,7 +123,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(1, "netns failure"), + netnsClient: netns.NewMock(1, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -141,7 +141,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(2, "netns failure"), + netnsClient: netns.NewMock(2, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -159,7 +159,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(3, "netns failure"), + netnsClient: netns.NewMock(3, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -200,7 +200,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -217,7 +217,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -235,7 +235,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -279,7 +279,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -304,7 +304,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(5, "netns failure"), + netnsClient: netns.NewMock(5, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -359,7 +359,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -384,7 +384,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -417,7 +417,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -443,7 +443,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -490,7 +490,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -516,7 +516,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), From 095fdc956d708ef939dbe4fc2ef0cd1b1574457a Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 10:23:19 -0700 Subject: [PATCH 31/86] Fix tests, lint --- netns/mocknetns.go | 16 ++++----- network/native_endpointclient_linux.go | 40 ++++++++++----------- network/native_endpointclient_linux_test.go | 28 +++++++-------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index 4026de8ab0..940d2b1cf3 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -4,10 +4,10 @@ import ( "github.com/pkg/errors" ) -var ErrorMockNetns = errors.New("mock netns error") +var ErrorMock = errors.New("mock netns error") -func newErrorMockNetns(errStr string) error { - return errors.Wrap(ErrorMockNetns, errStr) +func newErrorMock(errStr string) error { + return errors.Wrap(ErrorMock, errStr) } type MockNetns struct { @@ -24,31 +24,31 @@ func NewMock(failMethod int, failMessage string) *MockNetns { func (f *MockNetns) Get() (int, error) { if f.failMethod == 1 { - return 0, newErrorMockNetns(f.failMessage) + return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) GetFromName(name string) (int, error) { if f.failMethod == 2 { - return 0, newErrorMockNetns(f.failMessage) + return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) Set(handle int) error { if f.failMethod == 3 { - return newErrorMockNetns(f.failMessage) + return newErrorMock(f.failMessage) } return nil } func (f *MockNetns) NewNamed(name string) (int, error) { if f.failMethod == 4 { - return 0, newErrorMockNetns(f.failMessage) + return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) DeleteNamed(name string) error { if f.failMethod == 5 { - return newErrorMockNetns(f.failMessage) + return newErrorMock(f.failMessage) } return nil } diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 7b0cd6d4e7..6ebcb709bb 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -22,10 +22,10 @@ const ( ) type NativeEndpointClient struct { - eth0VethName string //So like eth0 - ethXVethName string //So like eth0.X - vnetVethName string //Peer is containerVethName - containerVethName string //Peer is vnetVethName + eth0VethName string // So like eth0 + ethXVethName string // So like eth0.X + vnetVethName string // Peer is containerVethName + containerVethName string // Peer is vnetVethName vnetMac net.HardwareAddr containerMac net.HardwareAddr @@ -66,25 +66,24 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { returnedTo, err := GetCurrentThreadNamespace() if err != nil { return errors.Wrap(err, "failed to get vm ns") - } else { - log.Printf("[native] VM Namespace: %s", returnedTo.file.Name()) } + log.Printf("[native] VM Namespace: %s", returnedTo.file.Name()) log.Printf("[native] Checking if NS exists...") vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) // If the ns does not exist, the below code will trigger to create it - if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { + // Something else went wrong return errors.Wrap(existingErr, "error other than vnet ns doesn't exist") - } else { - log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") - vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) - if err != nil { - return errors.Wrap(err, "failed to create vnet ns") - } - } + // The vnet ns does not exist, which is okay + log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") + vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) + if err != nil { + return errors.Wrap(err, "failed to create vnet ns") + } + } else { log.Printf("[native] Existing NS detected.") } @@ -115,11 +114,12 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { ethXCreated := true if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { + // Something else went wrong return errors.Wrap(err, "error other than eth0.x already exists") - } else { - log.Printf("[native] eth0.X already exists") - ethXCreated = false } + // The interface exists already, which is okay + log.Printf("[native] eth0.X already exists") + ethXCreated = false } if ethXCreated { log.Printf("[native] Move vlan link (eth0.X) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) @@ -224,10 +224,10 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(ep } if err := client.AddDefaultRoutes(client.containerVethName); err != nil { - return errors.Wrap(err, "failed Container ns add default routes") + return errors.Wrap(err, "failed container ns add default routes") } if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { - return errors.Wrap(err, "failed Container ns add default arp") + return errors.Wrap(err, "failed container ns add default arp") } return nil } @@ -352,7 +352,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) if delErr != nil { - return errors.Wrap(err, "failed to delete namespace") + return errors.Wrap(delErr, "failed to delete namespace") } } return nil diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 3a2f9d71b1..756b11257a 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -77,7 +77,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to move vnetVethName into vnet NS, deleting : " + netlink.ErrorMockNetlink.Error() + " : netlink fail", + wantErrMsg: "failed to move vnetVethName into vnet ns, deleting: " + netlink.ErrorMockNetlink.Error() + " : netlink fail", }, { name: "Add endpoints get interface fail for primary interface (eth0)", @@ -95,7 +95,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to get eth0 interface : " + netio.ErrMockNetIOFail.Error() + ":eth0", + wantErrMsg: "failed to get eth0 interface: " + netio.ErrMockNetIOFail.Error() + ":eth0", }, { name: "Add endpoints get interface fail for getting container veth", @@ -113,7 +113,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Container veth does not exist : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + wantErrMsg: "container veth does not exist: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, { name: "Add endpoints NetNS Get fail", @@ -131,7 +131,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to get VM NS handle : " + netns.ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "failed to get vm ns handle: netns failure: " + netns.ErrorMock.Error(), }, { name: "Add endpoints NetNS GetFromName fail (with error other than file does not exists)", @@ -149,7 +149,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Error other than vnet NS doesn't exist : " + netns.ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "error other than vnet ns doesn't exist: netns failure: " + netns.ErrorMock.Error(), }, { name: "Add endpoints NetNS Set fail", @@ -167,7 +167,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to set current NS to VM : " + netns.ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "failed to set current ns to vm: netns failure: " + netns.ErrorMock.Error(), }, } @@ -177,7 +177,7 @@ func TestNativeAddEndpoints(t *testing.T) { err := tt.client.PopulateVM(tt.epInfo) if tt.wantErr { require.Error(t, err) - require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) } else { require.NoError(t, err) } @@ -225,7 +225,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : eth0.X doesn't exist : " + netio.ErrMockNetIOFail.Error() + ":eth0.1", + wantErrMsg: "eth0.x doesn't exist: " + netio.ErrMockNetIOFail.Error() + ":eth0.1", }, { name: "Add endpoints fail check vnet veth exists", @@ -243,7 +243,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : vnet veth doesn't exist : " + netio.ErrMockNetIOFail.Error() + ":A1veth0", + wantErrMsg: "vnet veth doesn't exist: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", }, } @@ -253,7 +253,7 @@ func TestNativeAddEndpoints(t *testing.T) { err := tt.client.PopulateVnet(tt.epInfo) if tt.wantErr { require.Error(t, err) - require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) } else { require.NoError(t, err) } @@ -319,7 +319,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to delete namespace : " + netns.ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "failed to delete namespace: netns failure: " + netns.ErrorMock.Error(), }, } @@ -329,7 +329,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { err := tt.client.DeleteEndpointsImpl(tt.ep) if tt.wantErr { require.Error(t, err) - require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) } else { require.NoError(t, err) } @@ -458,7 +458,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed Container NS add default routes : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + wantErrMsg: "failed container ns add default routes: addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, } @@ -531,7 +531,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed adding routes to vnet specific to this container : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", + wantErrMsg: "failed adding routes to vnet specific to this container: addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", }, } From 4a634712eb81b4c407de6dbcac90fce71c355c46 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 13:30:02 -0700 Subject: [PATCH 32/86] Fixes from linter Works on VMSS --- network/native_endpointclient_linux.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 6ebcb709bb..2041ad892b 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -73,6 +73,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) // If the ns does not exist, the below code will trigger to create it if existingErr != nil { + log.Printf("[native] Existing response is: %s", existingErr.Error()) if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { // Something else went wrong return errors.Wrap(existingErr, "error other than vnet ns doesn't exist") @@ -113,6 +114,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { existingErr = vishnetlink.LinkAdd(link) ethXCreated := true if existingErr != nil { + log.Printf("[native] Existing response is: %s", existingErr.Error()) if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { // Something else went wrong return errors.Wrap(err, "error other than eth0.x already exists") @@ -166,14 +168,14 @@ func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { return nil } func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { - //There are no rules to add here - //Described as rules on ip addresses on the container interface + // There are no rules to add here + // Described as rules on ip addresses on the container interface return nil } func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { - //Never added any endpoint rules + // Never added any endpoint rules } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { @@ -313,7 +315,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { // Helper that creates arp entry for the current NS which maps the virtual // gateway (169.254.1.1) to destMac on a particular interfaceName // Example: (169.254.1.1) at 12:34:56:78:9a:bc [ether] PERM on -func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { +func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) error { _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", virtualGwNet.String(), destMac) From c291f1c5ab7257979cdfd942939bfdddcb8adf8e Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 13:49:56 -0700 Subject: [PATCH 33/86] Replacing references to ethX with vlan veth --- network/endpoint_linux.go | 16 ++++---- network/native_endpointclient_linux.go | 42 ++++++++++----------- network/native_endpointclient_linux_test.go | 38 +++++++++---------- 3 files changed, 47 insertions(+), 49 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index d035c4990c..9dbd21bfec 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -59,8 +59,6 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p var localIP string var epClient EndpointClient var vlanid int = 0 - var ethXIfName string - var vnetNSName string if nw.Endpoints[epInfo.Id] != nil { log.Printf("[net] Endpoint alreday exists.") @@ -94,16 +92,16 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p if vlanid != 0 { if nw.Mode == opModeNative { log.Printf("Native client") - ethXIfName = fmt.Sprintf("eth0.%d", vlanid) - vnetNSName = fmt.Sprintf("az_ns_%d", vlanid) + vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, vlanid) + vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) epClient = &NativeEndpointClient{ eth0VethName: nw.extIf.Name, - ethXVethName: ethXIfName, + vlanVethName: vlanVethName, vnetVethName: hostIfName, containerVethName: contIfName, vnetNSName: vnetNSName, - mode: nw.Mode, + nw: nw, vlanID: vlanid, netnsClient: netns.New(), netlink: nl, @@ -265,16 +263,16 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. epInfo := ep.getInfo() if nw.Mode == opModeNative { log.Printf("Native client") - ethXIfName := fmt.Sprintf("eth0.%d", ep.VlanID) + vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, ep.VlanID) vnetNSName := fmt.Sprintf("az_ns_%d", ep.VlanID) epClient = &NativeEndpointClient{ eth0VethName: nw.extIf.Name, - ethXVethName: ethXIfName, + vlanVethName: vlanVethName, vnetVethName: ep.HostIfName, containerVethName: "", vnetNSName: vnetNSName, - mode: nw.Mode, + nw: nw, vlanID: ep.VlanID, netnsClient: netns.New(), netlink: nl, diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 2041ad892b..dec7c0c669 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -23,18 +23,18 @@ const ( type NativeEndpointClient struct { eth0VethName string // So like eth0 - ethXVethName string // So like eth0.X + vlanVethName string // So like eth0.1 vnetVethName string // Peer is containerVethName containerVethName string // Peer is vnetVethName vnetMac net.HardwareAddr containerMac net.HardwareAddr - ethXMac net.HardwareAddr + vlanMac net.HardwareAddr vnetNSName string vnetNSFileDescriptor int - mode string + nw *network vlanID int netnsClient NetnsInterface netlink netlink.NetlinkInterface @@ -97,7 +97,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) linkAttrs := vishnetlink.NewLinkAttrs() - linkAttrs.Name = client.ethXVethName + linkAttrs.Name = client.vlanVethName // Get parent interface index. Index is consistent across libraries. eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) if err != nil { @@ -109,27 +109,27 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { LinkAttrs: linkAttrs, VlanId: client.vlanID, } - log.Printf("[native] Attempting to create eth0.X link in VM NS") - // Attempting to create ethX + log.Printf("[native] Attempting to create %s link in VM NS", client.vlanVethName) + // Attempting to create vlan veth existingErr = vishnetlink.LinkAdd(link) - ethXCreated := true + vlanVethCreated := true if existingErr != nil { log.Printf("[native] Existing response is: %s", existingErr.Error()) if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { // Something else went wrong - return errors.Wrap(err, "error other than eth0.x already exists") + return errors.Wrap(err, "error other than vlan veth already exists") } // The interface exists already, which is okay - log.Printf("[native] eth0.X already exists") - ethXCreated = false + log.Printf("[native] %s already exists", client.vlanVethName) + vlanVethCreated = false } - if ethXCreated { - log.Printf("[native] Move vlan link (eth0.X) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) - if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { + if vlanVethCreated { + log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) + if err = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { - log.Errorf("deleting ethx veth failed on addendpoint failure:%v", delErr) + log.Errorf("deleting vlan veth failed on addendpoint failure:%v", delErr) } - return errors.Wrap(err, "deleting ethx veth in vm ns due to addendpoint failure") + return errors.Wrap(err, "deleting vlan veth in vm ns due to addendpoint failure") } } @@ -154,11 +154,11 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Called from AddEndpoints, Namespace: Vnet func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { - ethXVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) + vlanVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vlanVethName) if err != nil { - return errors.Wrap(err, "eth0.x doesn't exist") + return errors.Wrap(err, "vlan veth doesn't exist") } - client.ethXMac = ethXVethIf.HardwareAddr + client.vlanMac = vlanVethIf.HardwareAddr vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) if err != nil { @@ -245,10 +245,10 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo // Add route specifying which device the pod ip(s) are on routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) - if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { + if err = client.AddDefaultRoutes(client.vlanVethName); err != nil { return errors.Wrap(err, "failed vnet ns add default/gateway routes (indempotent)") } - if err = client.AddDefaultArp(client.ethXVethName, azureMac); err != nil { + if err = client.AddDefaultArp(client.vlanVethName, azureMac); err != nil { return errors.Wrap(err, "failed vnet ns add default arp entry (idempotent)") } if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { @@ -349,7 +349,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { } log.Printf("[native] There are %d routes remaining: %v", len(routes), routes) if len(routes) <= numDefaultRoutes { - // Deletes default arp, default routes, ethX; there are two default routes + // Deletes default arp, default routes, vlan veth; there are two default routes // so when we have <= numDefaultRoutes routes left, no containers use this namespace log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 756b11257a..15c462d29d 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -31,7 +31,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints no existing vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -48,7 +48,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints with existing vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -65,7 +65,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints netlink fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -83,7 +83,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints get interface fail for primary interface (eth0)", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -101,7 +101,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints get interface fail for getting container veth", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -119,7 +119,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints NetNS Get fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -137,7 +137,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints NetNS GetFromName fail (with error other than file does not exists)", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -155,7 +155,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints NetNS Set fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -196,7 +196,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints second half", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -213,7 +213,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints fail check eth0.X exists", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -231,7 +231,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints fail check vnet veth exists", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -275,7 +275,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { name: "Delete endpoint good path", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -300,7 +300,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { name: "Delete endpoint fail to delete namespace", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -354,7 +354,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes good path for container", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -379,7 +379,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes multiple IPs", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -412,7 +412,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes assign ip fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -438,7 +438,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes container 2nd default route added fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -485,7 +485,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes good path for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -511,7 +511,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes fail final routes for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", From 97022b88cae6f9c60bfd0dda15bc96b09caf4b25 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 13:53:35 -0700 Subject: [PATCH 34/86] Removed unnecessary log --- network/native_endpointclient_linux.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index dec7c0c669..573d5fd924 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -62,12 +62,6 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { if err != nil { return errors.Wrap(err, "failed to get vm ns handle") } - // Inform current namespace - returnedTo, err := GetCurrentThreadNamespace() - if err != nil { - return errors.Wrap(err, "failed to get vm ns") - } - log.Printf("[native] VM Namespace: %s", returnedTo.file.Name()) log.Printf("[native] Checking if NS exists...") vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) From c40d4e8309cfd6ef25c6c6d8633f4d1f5ce1fea1 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 13:55:03 -0700 Subject: [PATCH 35/86] Removed unnecessary mac, fix tests --- network/native_endpointclient_linux.go | 5 +---- network/native_endpointclient_linux_test.go | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 573d5fd924..b5c359d3a0 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -29,7 +29,6 @@ type NativeEndpointClient struct { vnetMac net.HardwareAddr containerMac net.HardwareAddr - vlanMac net.HardwareAddr vnetNSName string vnetNSFileDescriptor int @@ -148,12 +147,10 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Called from AddEndpoints, Namespace: Vnet func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { - vlanVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vlanVethName) + _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanVethName) if err != nil { return errors.Wrap(err, "vlan veth doesn't exist") } - client.vlanMac = vlanVethIf.HardwareAddr - vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) if err != nil { return errors.Wrap(err, "vnet veth doesn't exist") diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 15c462d29d..da708bae2d 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -193,7 +193,7 @@ func TestNativeAddEndpoints(t *testing.T) { }{ // Populate the client with information from the vnet and set up vnet { - name: "Add endpoints second half", + name: "Add endpoints get vnet veth mac address", client: &NativeEndpointClient{ eth0VethName: "eth0", vlanVethName: "eth0.1", @@ -210,7 +210,7 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr: false, }, { - name: "Add endpoints fail check eth0.X exists", + name: "Add endpoints fail check vlan veth exists", client: &NativeEndpointClient{ eth0VethName: "eth0", vlanVethName: "eth0.1", @@ -225,7 +225,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "eth0.x doesn't exist: " + netio.ErrMockNetIOFail.Error() + ":eth0.1", + wantErrMsg: "vlan veth doesn't exist: " + netio.ErrMockNetIOFail.Error() + ":eth0.1", }, { name: "Add endpoints fail check vnet veth exists", From c58059bc6d3e571b2369ad555836be39b1621dfb Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 14:11:30 -0700 Subject: [PATCH 36/86] Mockns method name enum --- netns/mocknetns.go | 18 +++++++++++++----- network/native_endpointclient_linux_test.go | 10 +++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index 940d2b1cf3..00122be9a0 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -4,6 +4,14 @@ import ( "github.com/pkg/errors" ) +const ( + Get int = 1 + GetFromName int = 2 + Set int = 3 + NewNamed int = 4 + DeleteNamed int = 5 +) + var ErrorMock = errors.New("mock netns error") func newErrorMock(errStr string) error { @@ -23,31 +31,31 @@ func NewMock(failMethod int, failMessage string) *MockNetns { } func (f *MockNetns) Get() (int, error) { - if f.failMethod == 1 { + if f.failMethod == Get { return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) GetFromName(name string) (int, error) { - if f.failMethod == 2 { + if f.failMethod == GetFromName { return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) Set(handle int) error { - if f.failMethod == 3 { + if f.failMethod == Set { return newErrorMock(f.failMessage) } return nil } func (f *MockNetns) NewNamed(name string) (int, error) { - if f.failMethod == 4 { + if f.failMethod == NewNamed { return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) DeleteNamed(name string) error { - if f.failMethod == 5 { + if f.failMethod == DeleteNamed { return newErrorMock(f.failMessage) } return nil diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index da708bae2d..cdce9bfb43 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -35,7 +35,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(2, "no such file or directory"), + netnsClient: netns.NewMock(netns.GetFromName, "no such file or directory"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -123,7 +123,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(1, "netns failure"), + netnsClient: netns.NewMock(netns.Get, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -141,7 +141,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(2, "netns failure"), + netnsClient: netns.NewMock(netns.GetFromName, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -159,7 +159,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(3, "netns failure"), + netnsClient: netns.NewMock(netns.Set, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -304,7 +304,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(5, "netns failure"), + netnsClient: netns.NewMock(netns.DeleteNamed, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), From a724dc573a3bef0f642d284bc7a6fc9d01a54475 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 15:20:59 -0700 Subject: [PATCH 37/86] Unable to use GetNetworkInterfaceByName due to NS If I use GetNetworkInterface, I need to be in the vnet NS, but that means I will need to call ExecuteInNS, which causes tests to fail. --- network/native_endpointclient_linux.go | 28 ++++++++++---------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index b5c359d3a0..ef901591bd 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -11,7 +11,6 @@ import ( "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" "github.com/pkg/errors" - vishnetlink "github.com/vishvananda/netlink" ) @@ -89,13 +88,15 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { } log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) - linkAttrs := vishnetlink.NewLinkAttrs() - linkAttrs.Name = client.vlanVethName + // Get parent interface index. Index is consistent across libraries. eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) if err != nil { return errors.Wrap(err, "failed to get eth0 interface") } + + linkAttrs := vishnetlink.NewLinkAttrs() + linkAttrs.Name = client.vlanVethName // Set the peer linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ @@ -103,20 +104,10 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { VlanId: client.vlanID, } log.Printf("[native] Attempting to create %s link in VM NS", client.vlanVethName) - // Attempting to create vlan veth + // Create vlan veth existingErr = vishnetlink.LinkAdd(link) - vlanVethCreated := true - if existingErr != nil { - log.Printf("[native] Existing response is: %s", existingErr.Error()) - if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { - // Something else went wrong - return errors.Wrap(err, "error other than vlan veth already exists") - } - // The interface exists already, which is okay - log.Printf("[native] %s already exists", client.vlanVethName) - vlanVethCreated = false - } - if vlanVethCreated { + if existingErr == nil { + // vlan veth was created successfully, so move the vlan veth you created log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) if err = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { @@ -124,6 +115,9 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { } return errors.Wrap(err, "deleting vlan veth in vm ns due to addendpoint failure") } + } else { + // Otherwise, no need to create the vlan veth nor move it anywhere + log.Printf("[native] %s already exists", client.vlanVethName) } if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { @@ -242,7 +236,7 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo if err = client.AddDefaultArp(client.vlanVethName, azureMac); err != nil { return errors.Wrap(err, "failed vnet ns add default arp entry (idempotent)") } - if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { + if err = addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return errors.Wrap(err, "failed adding routes to vnet specific to this container") } // Return to ConfigureContainerInterfacesAndRoutes From 062aa9fc6c7c48f9b0b35a0438802fe721070a4e Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 15:34:09 -0700 Subject: [PATCH 38/86] Fixes from linter --- netns/netns.go | 15 +++++++++------ network/native_endpointclient_linux.go | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/netns/netns.go b/netns/netns.go index 1389db124b..86b80a73e2 100644 --- a/netns/netns.go +++ b/netns/netns.go @@ -1,6 +1,9 @@ package netns -import "github.com/vishvananda/netns" +import ( + "github.com/pkg/errors" + "github.com/vishvananda/netns" +) type Netns struct{} @@ -10,19 +13,19 @@ func New() *Netns { func (f *Netns) Get() (int, error) { nsHandle, err := netns.Get() - return int(nsHandle), err + return int(nsHandle), errors.Wrap(err, "netns impl") } func (f *Netns) GetFromName(name string) (int, error) { nsHandle, err := netns.GetFromName(name) - return int(nsHandle), err + return int(nsHandle), errors.Wrap(err, "netns impl") } func (f *Netns) Set(fileDescriptor int) error { - return netns.Set(netns.NsHandle(fileDescriptor)) + return errors.Wrap(netns.Set(netns.NsHandle(fileDescriptor)), "netns impl") } func (f *Netns) NewNamed(name string) (int, error) { nsHandle, err := netns.NewNamed(name) - return int(nsHandle), err + return int(nsHandle), errors.Wrap(err, "netns impl") } func (f *Netns) DeleteNamed(name string) error { - return netns.DeleteNamed(name) + return errors.Wrap(netns.DeleteNamed(name), "netns impl") } diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index ef901591bd..6c573b0c1a 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -246,7 +246,7 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo // Helper that gets the routes in the vnet NS for a particular list of IP addresses // Example: 192.168.0.4 dev proto static func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []RouteInfo { - var routeInfoList []RouteInfo + routeInfoList := make([]RouteInfo, 0, len(ipAddresses)) // Add route specifying which device the pod ip(s) are on for _, ipAddr := range ipAddresses { var ( @@ -306,7 +306,7 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) virtualGwNet.String(), destMac) hardwareAddr, err := net.ParseMAC(destMac) if err != nil { - return err + return errors.Wrap(err, "unable to parse mac") } if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, interfaceName, From a0676b22a90bd1e7325179f668047dc43efd40d5 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 11:02:57 -0700 Subject: [PATCH 39/86] Assume if NS exists, vlan veth exists Tests ok --- netns/mocknetns.go | 14 +-- network/native_endpointclient_linux.go | 95 +++++++++++---------- network/native_endpointclient_linux_test.go | 63 +++++--------- 3 files changed, 81 insertions(+), 91 deletions(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index 00122be9a0..5f4eed6843 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -20,42 +20,44 @@ func newErrorMock(errStr string) error { type MockNetns struct { failMethod int + failMethod2 int failMessage string } -func NewMock(failMethod int, failMessage string) *MockNetns { +func NewMock(failMethod int, failMethod2 int, failMessage string) *MockNetns { return &MockNetns{ failMethod: failMethod, + failMethod2: failMethod2, failMessage: failMessage, } } func (f *MockNetns) Get() (int, error) { - if f.failMethod == Get { + if f.failMethod == Get || f.failMethod2 == Get { return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) GetFromName(name string) (int, error) { - if f.failMethod == GetFromName { + if f.failMethod == GetFromName || f.failMethod2 == GetFromName { return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) Set(handle int) error { - if f.failMethod == Set { + if f.failMethod == Set || f.failMethod2 == Set { return newErrorMock(f.failMessage) } return nil } func (f *MockNetns) NewNamed(name string) (int, error) { - if f.failMethod == NewNamed { + if f.failMethod == NewNamed || f.failMethod2 == NewNamed { return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) DeleteNamed(name string) error { - if f.failMethod == DeleteNamed { + if f.failMethod == DeleteNamed || f.failMethod2 == DeleteNamed { return newErrorMock(f.failMessage) } return nil diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 6c573b0c1a..b4e6e695e3 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -3,7 +3,6 @@ package network import ( "fmt" "net" - "strings" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" @@ -64,61 +63,67 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { log.Printf("[native] Checking if NS exists...") vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) // If the ns does not exist, the below code will trigger to create it + // This will also (we assume) mean the vlan veth does not exist if existingErr != nil { - log.Printf("[native] Existing response is: %s", existingErr.Error()) - if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { - // Something else went wrong - return errors.Wrap(existingErr, "error other than vnet ns doesn't exist") - } - // The vnet ns does not exist, which is okay + // We assume the only possible error is that the namespace doesn't exist log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) if err != nil { return errors.Wrap(err, "failed to create vnet ns") } + client.vnetNSFileDescriptor = vnetNS + deleteNSIfNotNilErr := client.netnsClient.Set(vmNS) + // Any failure will trigger removing the namespace created + defer func() { + if deleteNSIfNotNilErr != nil { + log.Logf("[native] Removing vnet ns due to failure...") + err = client.netnsClient.DeleteNamed(client.vnetNSName) + if err != nil { + log.Errorf("failed to cleanup/delete ns after failing to create vlan veth") + } + } + }() + if deleteNSIfNotNilErr != nil { + return errors.Wrap(deleteNSIfNotNilErr, "failed to set current ns to vm") + } - } else { - log.Printf("[native] Existing NS detected.") - } - client.vnetNSFileDescriptor = vnetNS - - err = client.netnsClient.Set(vmNS) - if err != nil { - return errors.Wrap(err, "failed to set current ns to vm") - } - - log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) - - // Get parent interface index. Index is consistent across libraries. - eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) - if err != nil { - return errors.Wrap(err, "failed to get eth0 interface") - } - - linkAttrs := vishnetlink.NewLinkAttrs() - linkAttrs.Name = client.vlanVethName - // Set the peer - linkAttrs.ParentIndex = eth0.Index - link := &vishnetlink.Vlan{ - LinkAttrs: linkAttrs, - VlanId: client.vlanID, - } - log.Printf("[native] Attempting to create %s link in VM NS", client.vlanVethName) - // Create vlan veth - existingErr = vishnetlink.LinkAdd(link) - if existingErr == nil { - // vlan veth was created successfully, so move the vlan veth you created - log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) - if err = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { - if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { - log.Errorf("deleting vlan veth failed on addendpoint failure:%v", delErr) + // Now create vlan veth + log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) + // Get parent interface index. Index is consistent across libraries. + eth0, deleteNSIfNotNilErr := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) + if deleteNSIfNotNilErr != nil { + return errors.Wrap(deleteNSIfNotNilErr, "failed to get eth0 interface") + } + linkAttrs := vishnetlink.NewLinkAttrs() + linkAttrs.Name = client.vlanVethName + // Set the peer + linkAttrs.ParentIndex = eth0.Index + link := &vishnetlink.Vlan{ + LinkAttrs: linkAttrs, + VlanId: client.vlanID, + } + log.Printf("[native] Attempting to create %s link in VM NS", client.vlanVethName) + // Create vlan veth + deleteNSIfNotNilErr = vishnetlink.LinkAdd(link) + if deleteNSIfNotNilErr == nil { + // vlan veth was created successfully, so move the vlan veth you created + log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) + deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) + if deleteNSIfNotNilErr != nil { + if delErr := client.netlink.DeleteLink(client.vlanVethName); delErr != nil { + log.Errorf("deleting vlan veth failed on addendpoint failure") + } + return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") } - return errors.Wrap(err, "deleting vlan veth in vm ns due to addendpoint failure") + } else { + // Any other error + return errors.Wrap(deleteNSIfNotNilErr, "failed to create vlan vnet link after making new ns") } + } else { - // Otherwise, no need to create the vlan veth nor move it anywhere - log.Printf("[native] %s already exists", client.vlanVethName) + log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanVethName) } + client.vnetNSFileDescriptor = vnetNS if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { return errors.Wrap(err, "failed to create veth pair") diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index cdce9bfb43..9aa2fcbd4e 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -28,21 +28,22 @@ func TestNativeAddEndpoints(t *testing.T) { }{ // Populating VM with data and creating interfaces/links { - name: "Add endpoints no existing vnet ns", + name: "Add endpoints create vnet ns failure", client: &NativeEndpointClient{ eth0VethName: "eth0", vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.GetFromName, "no such file or directory"), + netnsClient: netns.NewMock(netns.GetFromName, netns.NewNamed, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, - epInfo: &EndpointInfo{}, - wantErr: false, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "failed to create vnet ns: netns failure: " + netns.ErrorMock.Error(), }, { name: "Add endpoints with existing vnet ns", @@ -52,7 +53,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -69,7 +70,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -87,7 +88,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(netns.GetFromName, 0, "netns fail"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -105,11 +106,11 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 2), + netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, wantErr: true, @@ -123,7 +124,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.Get, "netns failure"), + netnsClient: netns.NewMock(netns.Get, 0, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -133,24 +134,6 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr: true, wantErrMsg: "failed to get vm ns handle: netns failure: " + netns.ErrorMock.Error(), }, - { - name: "Add endpoints NetNS GetFromName fail (with error other than file does not exists)", - client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanVethName: "eth0.1", - vnetVethName: "A1veth0", - containerVethName: "B1veth0", - vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.GetFromName, "netns failure"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), - }, - epInfo: &EndpointInfo{}, - wantErr: true, - wantErrMsg: "error other than vnet ns doesn't exist: netns failure: " + netns.ErrorMock.Error(), - }, { name: "Add endpoints NetNS Set fail", client: &NativeEndpointClient{ @@ -159,7 +142,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.Set, "netns failure"), + netnsClient: netns.NewMock(netns.Set, netns.GetFromName, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -200,7 +183,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -217,7 +200,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -235,7 +218,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -279,7 +262,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -304,7 +287,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.DeleteNamed, "netns failure"), + netnsClient: netns.NewMock(netns.DeleteNamed, 0, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -359,7 +342,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -384,7 +367,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -417,7 +400,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -443,7 +426,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -490,7 +473,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -516,7 +499,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), From efdfb96eead135fc9c07083b74e334932fb122d1 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 12:05:53 -0700 Subject: [PATCH 40/86] Fixes for Linter --- netns/mocknetns.go | 6 +++++- netns/netns.go | 4 ++++ network/hnswrapper/hnsv1wrapper.go | 2 +- network/hnswrapper/hnsv1wrapperinterface.go | 2 +- network/hnswrapper/hnsv2wrapperfake.go | 2 +- network/native_endpointclient_linux.go | 7 +++++-- network/native_endpointclient_linux_test.go | 1 + 7 files changed, 18 insertions(+), 6 deletions(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index 5f4eed6843..f9a00821f9 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -24,7 +24,7 @@ type MockNetns struct { failMessage string } -func NewMock(failMethod int, failMethod2 int, failMessage string) *MockNetns { +func NewMock(failMethod, failMethod2 int, failMessage string) *MockNetns { return &MockNetns{ failMethod: failMethod, failMethod2: failMethod2, @@ -38,24 +38,28 @@ func (f *MockNetns) Get() (int, error) { } return 1, nil } + func (f *MockNetns) GetFromName(name string) (int, error) { if f.failMethod == GetFromName || f.failMethod2 == GetFromName { return 0, newErrorMock(f.failMessage) } return 1, nil } + func (f *MockNetns) Set(handle int) error { if f.failMethod == Set || f.failMethod2 == Set { return newErrorMock(f.failMessage) } return nil } + func (f *MockNetns) NewNamed(name string) (int, error) { if f.failMethod == NewNamed || f.failMethod2 == NewNamed { return 0, newErrorMock(f.failMessage) } return 1, nil } + func (f *MockNetns) DeleteNamed(name string) error { if f.failMethod == DeleteNamed || f.failMethod2 == DeleteNamed { return newErrorMock(f.failMessage) diff --git a/netns/netns.go b/netns/netns.go index 86b80a73e2..7937d02317 100644 --- a/netns/netns.go +++ b/netns/netns.go @@ -15,17 +15,21 @@ func (f *Netns) Get() (int, error) { nsHandle, err := netns.Get() return int(nsHandle), errors.Wrap(err, "netns impl") } + func (f *Netns) GetFromName(name string) (int, error) { nsHandle, err := netns.GetFromName(name) return int(nsHandle), errors.Wrap(err, "netns impl") } + func (f *Netns) Set(fileDescriptor int) error { return errors.Wrap(netns.Set(netns.NsHandle(fileDescriptor)), "netns impl") } + func (f *Netns) NewNamed(name string) (int, error) { nsHandle, err := netns.NewNamed(name) return int(nsHandle), errors.Wrap(err, "netns impl") } + func (f *Netns) DeleteNamed(name string) error { return errors.Wrap(netns.DeleteNamed(name), "netns impl") } diff --git a/network/hnswrapper/hnsv1wrapper.go b/network/hnswrapper/hnsv1wrapper.go index 247024220e..55c117347f 100644 --- a/network/hnswrapper/hnsv1wrapper.go +++ b/network/hnswrapper/hnsv1wrapper.go @@ -1,5 +1,5 @@ //go:build windows -//+build windows +// +build windows package hnswrapper diff --git a/network/hnswrapper/hnsv1wrapperinterface.go b/network/hnswrapper/hnsv1wrapperinterface.go index 6f7a3bda45..6367bead28 100644 --- a/network/hnswrapper/hnsv1wrapperinterface.go +++ b/network/hnswrapper/hnsv1wrapperinterface.go @@ -19,5 +19,5 @@ type HnsV1WrapperInterface interface { GetHNSEndpointByID(endpointID string) (*hcsshim.HNSEndpoint, error) HotAttachEndpoint(containerID string, endpointID string) error IsAttached(hnsep *hcsshim.HNSEndpoint, containerID string) (bool, error) - GetHNSGlobals() (*hcsshim.HNSGlobals, error) + GetHNSGlobals() (*hcsshim.HNSGlobals, error) } diff --git a/network/hnswrapper/hnsv2wrapperfake.go b/network/hnswrapper/hnsv2wrapperfake.go index 8e887812c4..4f59aa5aec 100644 --- a/network/hnswrapper/hnsv2wrapperfake.go +++ b/network/hnswrapper/hnsv2wrapperfake.go @@ -42,7 +42,7 @@ func NewHnsv2wrapperFake() *Hnsv2wrapperFake { } } -func delayHnsCall(delay time.Duration){ +func delayHnsCall(delay time.Duration) { time.Sleep(delay) } diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index b4e6e695e3..8edadfe8eb 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -157,6 +157,7 @@ func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { client.vnetMac = vnetVethIf.HardwareAddr return nil } + func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { // There are no rules to add here // Described as rules on ip addresses on the container interface @@ -167,12 +168,14 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { // Never added any endpoint rules } + func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { return errors.Wrap(err, "failed to move endpoint to container ns") } return nil } + func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return errors.Wrap(err, "failed to setup container interface") @@ -198,7 +201,6 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Container func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { - if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return errors.Wrap(err, "failed to assign ips to container veth interface") } @@ -226,7 +228,6 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(ep // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Vnet func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { - err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { return errors.Wrap(err, "failed to set loopback link state to up") @@ -322,11 +323,13 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) } return nil } + func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { return ExecuteInNS(client.vnetNSName, func() error { return client.DeleteEndpointsImpl(ep) }) } + func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 9aa2fcbd4e..c17ff6a22b 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -243,6 +243,7 @@ func TestNativeAddEndpoints(t *testing.T) { }) } } + func TestNativeDeleteEndpoints(t *testing.T) { nl := netlink.NewMockNetlink(false, "") plc := platform.NewMockExecClient(false) From 7349b8dc51e5f3e011487e509c73a98af3b0fc22 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 13:17:51 -0700 Subject: [PATCH 41/86] Snat refactor --- network/ovs_endpointclient_linux.go | 2 +- network/ovssnat/ovssnat_linux.go | 90 +++++++++++++-------------- network/ovssnat/ovssnat_linux_test.go | 4 +- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 54054f2572..4c62f24303 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -23,7 +23,7 @@ type OVSEndpointClient struct { hostPrimaryMac string containerVethName string containerMac string - snatClient ovssnat.OVSSnatClient + snatClient ovssnat.SnatClient infraVnetClient ovsinfravnet.OVSInfraVnetClient vlanID int enableSnatOnHost bool diff --git a/network/ovssnat/ovssnat_linux.go b/network/ovssnat/ovssnat_linux.go index 60c531fc67..f053de1ed1 100644 --- a/network/ovssnat/ovssnat_linux.go +++ b/network/ovssnat/ovssnat_linux.go @@ -30,13 +30,13 @@ const ( l2PreroutingEntries = "ebtables -t nat -L PREROUTING" ) -var errorOVSSnatClient = errors.New("OVSSnatClient Error") +var errorSnatClient = errors.New("OVSSnatClient Error") -func newErrorOVSSnatClient(errStr string) error { - return fmt.Errorf("%w : %s", errorOVSSnatClient, errStr) +func newErrorSnatClient(errStr string) error { + return fmt.Errorf("%w : %s", errorSnatClient, errStr) } -type OVSSnatClient struct { +type SnatClient struct { hostSnatVethName string hostPrimaryMac string containerSnatVethName string @@ -57,9 +57,9 @@ func NewSnatClient(hostIfName string, nl netlink.NetlinkInterface, ovsctlClient ovsctl.OvsInterface, plClient platform.ExecClient, -) OVSSnatClient { +) SnatClient { log.Printf("Initialize new snat client") - snatClient := OVSSnatClient{ + snatClient := SnatClient{ hostSnatVethName: hostIfName, containerSnatVethName: contIfName, localIP: localIP, @@ -77,7 +77,7 @@ func NewSnatClient(hostIfName string, return snatClient } -func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { +func (client *SnatClient) CreateSnatEndpoint(bridgeName string) error { // Create linux Bridge for outbound connectivity if err := client.createSnatBridge(client.snatBridgeIP, client.hostPrimaryMac, bridgeName); err != nil { log.Printf("creating snat bridge failed with error %v", err) @@ -100,31 +100,31 @@ func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { // Create veth pair to tie one end to container and other end to linux bridge if err := epc.CreateEndpoint(client.hostSnatVethName, client.containerSnatVethName); err != nil { log.Printf("Creating Snat Endpoint failed with error %v", err) - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } err := client.netlink.SetLinkMaster(client.hostSnatVethName, SnatBridgeName) if err != nil { - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } return nil } // AllowIPAddressesOnSnatBridge adds iptables rules that allows only specific Private IPs via linux bridge -func (client *OVSSnatClient) AllowIPAddressesOnSnatBridge() error { +func (client *SnatClient) AllowIPAddressesOnSnatBridge() error { if err := networkutils.AllowIPAddresses(SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { log.Printf("AllowIPAddresses failed with error %v", err) - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } return nil } // BlockIPAddressesOnSnatBridge adds iptables rules that blocks all private IPs flowing via linux bridge -func (client *OVSSnatClient) BlockIPAddressesOnSnatBridge() error { +func (client *SnatClient) BlockIPAddressesOnSnatBridge() error { if err := networkutils.BlockIPAddresses(SnatBridgeName, iptables.Append); err != nil { log.Printf("AllowIPAddresses failed with error %v", err) - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } return nil @@ -133,11 +133,11 @@ func (client *OVSSnatClient) BlockIPAddressesOnSnatBridge() error { /** Move container veth inside container network namespace **/ -func (client *OVSSnatClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { +func (client *SnatClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { log.Printf("[ovs] Setting link %v netns %v.", client.containerSnatVethName, netnsPath) err := client.netlink.SetLinkNetNs(client.containerSnatVethName, nsID) if err != nil { - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } return nil } @@ -145,10 +145,10 @@ func (client *OVSSnatClient) MoveSnatEndpointToContainerNS(netnsPath string, nsI /** Configure Routes and setup name for container veth **/ -func (client *OVSSnatClient) SetupSnatContainerInterface() error { +func (client *SnatClient) SetupSnatContainerInterface() error { epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) if err := epc.SetupContainerInterface(client.containerSnatVethName, azureSnatIfName); err != nil { - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } client.containerSnatVethName = azureSnatIfName @@ -156,7 +156,7 @@ func (client *OVSSnatClient) SetupSnatContainerInterface() error { return nil } -func getNCLocalAndGatewayIP(client *OVSSnatClient) (net.IP, net.IP) { +func getNCLocalAndGatewayIP(client *SnatClient) (net.IP, net.IP) { bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) containerIP, _, _ := net.ParseCIDR(client.localIP) return bridgeIP, containerIP @@ -165,19 +165,19 @@ func getNCLocalAndGatewayIP(client *OVSSnatClient) (net.IP, net.IP) { /** This function adds iptables rules that allows only host to NC communication and not the other way **/ -func (client *OVSSnatClient) AllowInboundFromHostToNC() error { +func (client *SnatClient) AllowInboundFromHostToNC() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Create CNI Ouptut chain if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } // Forward traffic from Ouptut chain to CNI Output chain if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err) - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } // Allow connection from Host to NC @@ -185,19 +185,19 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) if err != nil { log.Printf("AllowInboundFromHostToNC: Inserting output rule failed: %v", err) - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } // Create cniinput chain if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err) - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } // Forward from Input to cniinput chain if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIInputChain, err) - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } // Accept packets from NC only if established connection @@ -205,7 +205,7 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) if err != nil { log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err) - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } snatContainerVeth, _ := net.InterfaceByName(client.containerSnatVethName) @@ -215,13 +215,13 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error { err = client.netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr, false) if err != nil { log.Printf("AllowInboundFromHostToNC: Error adding static arp entry for ip %s mac %s: %v", containerIP, snatContainerVeth.HardwareAddr.String(), err) - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } return nil } -func (client *OVSSnatClient) DeleteInboundFromHostToNC() error { +func (client *SnatClient) DeleteInboundFromHostToNC() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Delete allow connection from Host to NC @@ -244,7 +244,7 @@ func (client *OVSSnatClient) DeleteInboundFromHostToNC() error { /** This function adds iptables rules that allows only NC to Host communication and not the other way **/ -func (client *OVSSnatClient) AllowInboundFromNCToHost() error { +func (client *SnatClient) AllowInboundFromNCToHost() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Create CNI Input chain @@ -299,7 +299,7 @@ func (client *OVSSnatClient) AllowInboundFromNCToHost() error { return err } -func (client *OVSSnatClient) DeleteInboundFromNCToHost() error { +func (client *SnatClient) DeleteInboundFromNCToHost() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Delete allow NC to Host connection @@ -323,28 +323,28 @@ func (client *OVSSnatClient) DeleteInboundFromNCToHost() error { Configures Local IP Address for container Veth **/ -func (client *OVSSnatClient) ConfigureSnatContainerInterface() error { +func (client *SnatClient) ConfigureSnatContainerInterface() error { log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.containerSnatVethName) ip, intIpAddr, _ := net.ParseCIDR(client.localIP) err := client.netlink.AddIPAddress(client.containerSnatVethName, ip, intIpAddr) if err != nil { - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } return nil } -func (client *OVSSnatClient) DeleteSnatEndpoint() error { +func (client *SnatClient) DeleteSnatEndpoint() error { log.Printf("[ovs] Deleting snat veth pair %v.", client.hostSnatVethName) err := client.netlink.DeleteLink(client.hostSnatVethName) if err != nil { log.Printf("[ovs] Failed to delete veth pair %v: %v.", client.hostSnatVethName, err) - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } return nil } -func (client *OVSSnatClient) setBridgeMac(hostPrimaryMac string) error { +func (client *SnatClient) setBridgeMac(hostPrimaryMac string) error { hwAddr, err := net.ParseMAC(hostPrimaryMac) if err != nil { log.Errorf("Error while parsing host primary mac: %s error:%+v", hostPrimaryMac, err) @@ -357,7 +357,7 @@ func (client *OVSSnatClient) setBridgeMac(hostPrimaryMac string) error { return err } -func (client *OVSSnatClient) dropArpForSnatBridgeApipaRange(snatBridgeIP, azSnatVethIfName string) error { +func (client *SnatClient) dropArpForSnatBridgeApipaRange(snatBridgeIP, azSnatVethIfName string) error { var err error _, ipCidr, _ := net.ParseCIDR(snatBridgeIP) if err = ebtables.SetArpDropRuleForIpCidr(ipCidr.String(), azSnatVethIfName); err != nil { @@ -370,7 +370,7 @@ func (client *OVSSnatClient) dropArpForSnatBridgeApipaRange(snatBridgeIP, azSnat /** This function creates linux bridge which will be used for outbound connectivity by NCs **/ -func (client *OVSSnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac string, mainInterface string) error { +func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac string, mainInterface string) error { _, err := net.InterfaceByName(SnatBridgeName) if err == nil { log.Printf("Snat Bridge already exists") @@ -385,7 +385,7 @@ func (client *OVSSnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMa } if err := client.netlink.AddLink(&link); err != nil { - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } } @@ -443,27 +443,27 @@ func (client *OVSSnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMa err = client.netlink.AddIPAddress(SnatBridgeName, ip, addr) if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { log.Printf("[net] Failed to add IP address %v: %v.", addr, err) - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } if err = client.netlink.SetLinkState(SnatBridgeName, true); err != nil { - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } if err = client.netlink.SetLinkState(azureSnatVeth0, true); err != nil { - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } if err = client.netlink.SetLinkMaster(azureSnatVeth0, SnatBridgeName); err != nil { - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } if err = client.netlink.SetLinkState(azureSnatVeth1, true); err != nil { - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } if err = client.ovsctlClient.AddPortOnOVSBridge(azureSnatVeth1, mainInterface, 0); err != nil { - return newErrorOVSSnatClient(err.Error()) + return newErrorSnatClient(err.Error()) } return nil @@ -472,7 +472,7 @@ func (client *OVSSnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMa /** This function adds iptable rules that will snat all traffic that has source ip in apipa range and coming via linux bridge **/ -func (client *OVSSnatClient) addMasqueradeRule(snatBridgeIPWithPrefix string) error { +func (client *SnatClient) addMasqueradeRule(snatBridgeIPWithPrefix string) error { _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) matchCondition := fmt.Sprintf("-s %s", ipNet.String()) return iptables.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) @@ -481,7 +481,7 @@ func (client *OVSSnatClient) addMasqueradeRule(snatBridgeIPWithPrefix string) er /** Drop all vlan traffic on linux bridge **/ -func (client *OVSSnatClient) addVlanDropRule() error { +func (client *SnatClient) addVlanDropRule() error { out, err := client.plClient.ExecuteCommand(l2PreroutingEntries) if err != nil { log.Printf("Error while listing ebtable rules %v", err) diff --git a/network/ovssnat/ovssnat_linux_test.go b/network/ovssnat/ovssnat_linux_test.go index 7717f06306..c7e6879da4 100644 --- a/network/ovssnat/ovssnat_linux_test.go +++ b/network/ovssnat/ovssnat_linux_test.go @@ -20,7 +20,7 @@ func TestMain(m *testing.M) { func TestAllowInboundFromHostToNC(t *testing.T) { nl := netlink.NewNetlink() - client := &OVSSnatClient{ + client := &SnatClient{ snatBridgeIP: "169.254.0.1/16", localIP: "169.254.0.4/16", containerSnatVethName: anyInterface, @@ -68,7 +68,7 @@ func TestAllowInboundFromHostToNC(t *testing.T) { func TestAllowInboundFromNCToHost(t *testing.T) { nl := netlink.NewNetlink() - client := &OVSSnatClient{ + client := &SnatClient{ snatBridgeIP: "169.254.0.1/16", localIP: "169.254.0.4/16", containerSnatVethName: anyInterface, From adb23b27e794712f5b3d7cf48eb068817f8c1272 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 13:19:56 -0700 Subject: [PATCH 42/86] Fix delete tests --- network/native_endpointclient_linux.go | 18 ++++++----- network/native_endpointclient_linux_test.go | 35 ++++++++++++++++++--- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 8edadfe8eb..4c9525db6d 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -326,22 +326,24 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { return ExecuteInNS(client.vnetNSName, func() error { - return client.DeleteEndpointsImpl(ep) + routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) + if err != nil { + return errors.Wrap(err, ("failed to get route list")) + } + log.Printf("[native] Routes remaining: %v", routes) + + return client.DeleteEndpointsImpl(ep, len(routes)) }) } -func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { +func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint, routesLeft int) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return errors.Wrap(err, "failed to remove routes") } + log.Printf("[native] There are %d routes remaining", routesLeft) - routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) - if err != nil { - return errors.Wrap(err, ("failed to get route list")) - } - log.Printf("[native] There are %d routes remaining: %v", len(routes), routes) - if len(routes) <= numDefaultRoutes { + if routesLeft <= numDefaultRoutes { // Deletes default arp, default routes, vlan veth; there are two default routes // so when we have <= numDefaultRoutes routes left, no containers use this namespace log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index c17ff6a22b..a6b8d471c8 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -254,9 +254,10 @@ func TestNativeDeleteEndpoints(t *testing.T) { ep *endpoint wantErr bool wantErrMsg string + routesLeft int }{ { - name: "Delete endpoint good path", + name: "Delete endpoint delete vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", vlanVethName: "eth0.1", @@ -277,9 +278,34 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, }, }, - wantErr: false, + routesLeft: numDefaultRoutes, + wantErr: false, + }, + { + name: "Delete endpoint do not delete vnet ns it is still in use", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + vlanVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: netns.NewMock(netns.DeleteNamed, 0, "netns failure"), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + ep: &endpoint{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + routesLeft: numDefaultRoutes + 1, + wantErr: false, }, - // You must have <= 2 ip routes on your machine for this to pass { name: "Delete endpoint fail to delete namespace", client: &NativeEndpointClient{ @@ -302,6 +328,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, }, }, + routesLeft: numDefaultRoutes, wantErr: true, wantErrMsg: "failed to delete namespace: netns failure: " + netns.ErrorMock.Error(), }, @@ -310,7 +337,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.DeleteEndpointsImpl(tt.ep) + err := tt.client.DeleteEndpointsImpl(tt.ep, tt.routesLeft) if tt.wantErr { require.Error(t, err) require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) From a775d939986626062b9cd3a70473bc3b67b7ca9d Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 13:41:35 -0700 Subject: [PATCH 43/86] Fix delete tests bug --- network/native_endpointclient_linux.go | 6 ++-- network/native_endpointclient_linux_test.go | 38 +++++++++------------ 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 4c9525db6d..d2cec73b7e 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -338,12 +338,12 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint, routesLeft int) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) + log.Printf("[native] There are %d routes remaining", routesLeft) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return errors.Wrap(err, "failed to remove routes") } - log.Printf("[native] There are %d routes remaining", routesLeft) - - if routesLeft <= numDefaultRoutes { + // We get the # of routes, THEN delete, and see if it adds up + if routesLeft <= numDefaultRoutes+len(routeInfoList) { // Deletes default arp, default routes, vlan veth; there are two default routes // so when we have <= numDefaultRoutes routes left, no containers use this namespace log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index a6b8d471c8..7ce99cb888 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -247,7 +247,16 @@ func TestNativeAddEndpoints(t *testing.T) { func TestNativeDeleteEndpoints(t *testing.T) { nl := netlink.NewMockNetlink(false, "") plc := platform.NewMockExecClient(false) - + IPAddresses := []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + { + IP: net.ParseIP("192.168.0.6"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + } tests := []struct { name string client *NativeEndpointClient @@ -271,14 +280,9 @@ func TestNativeDeleteEndpoints(t *testing.T) { netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ - IPAddresses: []net.IPNet{ - { - IP: net.ParseIP("192.168.0.4"), - Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), - }, - }, + IPAddresses: IPAddresses, }, - routesLeft: numDefaultRoutes, + routesLeft: numDefaultRoutes + len(IPAddresses), wantErr: false, }, { @@ -296,14 +300,9 @@ func TestNativeDeleteEndpoints(t *testing.T) { netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ - IPAddresses: []net.IPNet{ - { - IP: net.ParseIP("192.168.0.4"), - Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), - }, - }, + IPAddresses: IPAddresses, }, - routesLeft: numDefaultRoutes + 1, + routesLeft: numDefaultRoutes + len(IPAddresses) + 1, wantErr: false, }, { @@ -321,14 +320,9 @@ func TestNativeDeleteEndpoints(t *testing.T) { netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ - IPAddresses: []net.IPNet{ - { - IP: net.ParseIP("192.168.0.4"), - Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), - }, - }, + IPAddresses: IPAddresses, }, - routesLeft: numDefaultRoutes, + routesLeft: numDefaultRoutes + len(IPAddresses), wantErr: true, wantErrMsg: "failed to delete namespace: netns failure: " + netns.ErrorMock.Error(), }, From 4f750ef0ff60654740619e27ed2c82f0264e645b Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 15:03:48 -0700 Subject: [PATCH 44/86] More snat refactor --- network/ovs_endpoint_snatroute_linux.go | 8 +- network/ovs_endpointclient_linux.go | 4 +- network/snat/snat_linux.go | 500 ++++++++++++++++++++++++ network/snat/snat_linux_test.go | 114 ++++++ 4 files changed, 620 insertions(+), 6 deletions(-) create mode 100644 network/snat/snat_linux.go create mode 100644 network/snat/snat_linux_test.go diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index 5048bfc54a..5dbe1b9499 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/Azure/azure-container-networking/network/networkutils" - "github.com/Azure/azure-container-networking/network/ovssnat" + "github.com/Azure/azure-container-networking/network/snat" ) func NewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP string, epInfo *EndpointInfo) { @@ -12,7 +12,7 @@ func NewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP strin hostIfName := fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7]) contIfName := fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7]) - client.snatClient = ovssnat.NewSnatClient(hostIfName, + client.snatClient = snat.NewSnatClient(hostIfName, contIfName, localIP, snatBridgeIP, @@ -48,12 +48,12 @@ func AddSnatEndpointRules(client *OVSEndpointClient) error { } // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge - if err := AddStaticRoute(client.netlink, client.netioshim, ovssnat.ImdsIP, client.bridgeName); err != nil { + if err := AddStaticRoute(client.netlink, client.netioshim, snat.ImdsIP, client.bridgeName); err != nil { return err } nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) - if err := nuc.EnableIPForwarding(ovssnat.SnatBridgeName); err != nil { + if err := nuc.EnableIPForwarding(snat.SnatBridgeName); err != nil { return err } diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 4c62f24303..84dc19e27e 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -11,7 +11,7 @@ import ( "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/ovsinfravnet" - "github.com/Azure/azure-container-networking/network/ovssnat" + "github.com/Azure/azure-container-networking/network/snat" "github.com/Azure/azure-container-networking/ovsctl" "github.com/Azure/azure-container-networking/platform" ) @@ -23,7 +23,7 @@ type OVSEndpointClient struct { hostPrimaryMac string containerVethName string containerMac string - snatClient ovssnat.SnatClient + snatClient snat.SnatClient infraVnetClient ovsinfravnet.OVSInfraVnetClient vlanID int enableSnatOnHost bool diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go new file mode 100644 index 0000000000..a8b3ba0fbf --- /dev/null +++ b/network/snat/snat_linux.go @@ -0,0 +1,500 @@ +// Copyright 2017 Microsoft. All rights reserved. +// MIT License + +package snat + +import ( + "errors" + "fmt" + "net" + "strings" + + "github.com/Azure/azure-container-networking/ebtables" + "github.com/Azure/azure-container-networking/iptables" + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" +) + +const ( + azureSnatVeth0 = "azSnatveth0" + azureSnatVeth1 = "azSnatveth1" + azureSnatIfName = "eth1" + SnatBridgeName = "azSnatbr" + ImdsIP = "169.254.169.254/32" + vlanDropDeleteRule = "ebtables -t nat -D PREROUTING -p 802_1Q -j DROP" + vlanDropAddRule = "ebtables -t nat -A PREROUTING -p 802_1Q -j DROP" + vlanDropMatch = "-p 802_1Q -j DROP" + l2PreroutingEntries = "ebtables -t nat -L PREROUTING" +) + +var errorSnatClient = errors.New("OVSSnatClient Error") + +func newErrorSnatClient(errStr string) error { + return fmt.Errorf("%w : %s", errorSnatClient, errStr) +} + +type SnatClient struct { + hostSnatVethName string + hostPrimaryMac string + containerSnatVethName string + localIP string + snatBridgeIP string + SkipAddressesFromBlock []string + netlink netlink.NetlinkInterface + ovsctlClient ovsctl.OvsInterface + plClient platform.ExecClient +} + +func NewSnatClient(hostIfName string, + contIfName string, + localIP string, + snatBridgeIP string, + hostPrimaryMac string, + skipAddressesFromBlock []string, + nl netlink.NetlinkInterface, + ovsctlClient ovsctl.OvsInterface, + plClient platform.ExecClient, +) SnatClient { + log.Printf("Initialize new snat client") + snatClient := SnatClient{ + hostSnatVethName: hostIfName, + containerSnatVethName: contIfName, + localIP: localIP, + snatBridgeIP: snatBridgeIP, + hostPrimaryMac: hostPrimaryMac, + netlink: nl, + ovsctlClient: ovsctlClient, + plClient: plClient, + } + + snatClient.SkipAddressesFromBlock = append(snatClient.SkipAddressesFromBlock, skipAddressesFromBlock...) + + log.Printf("Initialize new snat client %+v", snatClient) + + return snatClient +} + +func (client *SnatClient) CreateSnatEndpoint(bridgeName string) error { + // Create linux Bridge for outbound connectivity + if err := client.createSnatBridge(client.snatBridgeIP, client.hostPrimaryMac, bridgeName); err != nil { + log.Printf("creating snat bridge failed with error %v", err) + return err + } + + // SNAT Rule to masquerade packets destined to non-vnet ip + if err := client.addMasqueradeRule(client.snatBridgeIP); err != nil { + log.Printf("Adding snat rule failed with error %v", err) + return err + } + + // Drop all vlan packets coming via linux bridge. + if err := client.addVlanDropRule(); err != nil { + log.Printf("Adding vlan drop rule failed with error %v", err) + return err + } + + epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + // Create veth pair to tie one end to container and other end to linux bridge + if err := epc.CreateEndpoint(client.hostSnatVethName, client.containerSnatVethName); err != nil { + log.Printf("Creating Snat Endpoint failed with error %v", err) + return newErrorSnatClient(err.Error()) + } + + err := client.netlink.SetLinkMaster(client.hostSnatVethName, SnatBridgeName) + if err != nil { + return newErrorSnatClient(err.Error()) + } + return nil +} + +// AllowIPAddressesOnSnatBridge adds iptables rules that allows only specific Private IPs via linux bridge +func (client *SnatClient) AllowIPAddressesOnSnatBridge() error { + if err := networkutils.AllowIPAddresses(SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { + log.Printf("AllowIPAddresses failed with error %v", err) + return newErrorSnatClient(err.Error()) + } + + return nil +} + +// BlockIPAddressesOnSnatBridge adds iptables rules that blocks all private IPs flowing via linux bridge +func (client *SnatClient) BlockIPAddressesOnSnatBridge() error { + if err := networkutils.BlockIPAddresses(SnatBridgeName, iptables.Append); err != nil { + log.Printf("AllowIPAddresses failed with error %v", err) + return newErrorSnatClient(err.Error()) + } + + return nil +} + +/** + Move container veth inside container network namespace +**/ +func (client *SnatClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { + log.Printf("[ovs] Setting link %v netns %v.", client.containerSnatVethName, netnsPath) + err := client.netlink.SetLinkNetNs(client.containerSnatVethName, nsID) + if err != nil { + return newErrorSnatClient(err.Error()) + } + return nil +} + +/** + Configure Routes and setup name for container veth +**/ +func (client *SnatClient) SetupSnatContainerInterface() error { + epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + if err := epc.SetupContainerInterface(client.containerSnatVethName, azureSnatIfName); err != nil { + return newErrorSnatClient(err.Error()) + } + + client.containerSnatVethName = azureSnatIfName + + return nil +} + +func getNCLocalAndGatewayIP(client *SnatClient) (net.IP, net.IP) { + bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) + containerIP, _, _ := net.ParseCIDR(client.localIP) + return bridgeIP, containerIP +} + +/** + This function adds iptables rules that allows only host to NC communication and not the other way +**/ +func (client *SnatClient) AllowInboundFromHostToNC() error { + bridgeIP, containerIP := getNCLocalAndGatewayIP(client) + + // Create CNI Ouptut chain + if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) + return newErrorSnatClient(err.Error()) + } + + // Forward traffic from Ouptut chain to CNI Output chain + if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err) + return newErrorSnatClient(err.Error()) + } + + // Allow connection from Host to NC + matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) + err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + if err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting output rule failed: %v", err) + return newErrorSnatClient(err.Error()) + } + + // Create cniinput chain + if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err) + return newErrorSnatClient(err.Error()) + } + + // Forward from Input to cniinput chain + if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIInputChain, err) + return newErrorSnatClient(err.Error()) + } + + // Accept packets from NC only if established connection + matchCondition = fmt.Sprintf(" -i %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related) + err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + if err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err) + return newErrorSnatClient(err.Error()) + } + + snatContainerVeth, _ := net.InterfaceByName(client.containerSnatVethName) + + // Add static arp entry for localIP to prevent arp going out of VM + log.Printf("Adding static arp entry for ip %s mac %s", containerIP, snatContainerVeth.HardwareAddr.String()) + err = client.netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr, false) + if err != nil { + log.Printf("AllowInboundFromHostToNC: Error adding static arp entry for ip %s mac %s: %v", containerIP, snatContainerVeth.HardwareAddr.String(), err) + return newErrorSnatClient(err.Error()) + } + + return nil +} + +func (client *SnatClient) DeleteInboundFromHostToNC() error { + bridgeIP, containerIP := getNCLocalAndGatewayIP(client) + + // Delete allow connection from Host to NC + matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) + err := iptables.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + if err != nil { + log.Printf("DeleteInboundFromHostToNC: Error removing output rule %v", err) + } + + // Remove static arp entry added for container local IP + log.Printf("Removing static arp entry for ip %s ", containerIP) + err = client.netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil, false) + if err != nil { + log.Printf("AllowInboundFromHostToNC: Error removing static arp entry for ip %s: %v", containerIP, err) + } + + return err +} + +/** + This function adds iptables rules that allows only NC to Host communication and not the other way +**/ +func (client *SnatClient) AllowInboundFromNCToHost() error { + bridgeIP, containerIP := getNCLocalAndGatewayIP(client) + + // Create CNI Input chain + if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err) + return err + } + + // Forward traffic from Input to cniinput chain + if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIInputChain, err) + return err + } + + // Allow NC to Host connection + matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) + err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + if err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting output rule failed: %v", err) + return err + } + + // Create CNI output chain + if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) + return err + } + + // Forward traffic from Output to CNI Output chain + if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err) + return err + } + + // Accept packets from Host only if established connection + matchCondition = fmt.Sprintf(" -o %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related) + err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) + if err != nil { + log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err) + return err + } + + snatContainerVeth, _ := net.InterfaceByName(client.containerSnatVethName) + + // Add static arp entry for localIP to prevent arp going out of VM + log.Printf("Adding static arp entry for ip %s mac %s", containerIP, snatContainerVeth.HardwareAddr.String()) + err = client.netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr, false) + if err != nil { + log.Printf("AllowInboundFromNCToHost: Error adding static arp entry for ip %s mac %s: %v", containerIP, snatContainerVeth.HardwareAddr.String(), err) + } + + return err +} + +func (client *SnatClient) DeleteInboundFromNCToHost() error { + bridgeIP, containerIP := getNCLocalAndGatewayIP(client) + + // Delete allow NC to Host connection + matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) + err := iptables.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) + if err != nil { + log.Printf("DeleteInboundFromNCToHost: Error removing output rule %v", err) + } + + // Remove static arp entry added for container local IP + log.Printf("Removing static arp entry for ip %s ", containerIP) + err = client.netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil, false) + if err != nil { + log.Printf("DeleteInboundFromNCToHost: Error removing static arp entry for ip %s: %v", containerIP, err) + } + + return err +} + +/** + Configures Local IP Address for container Veth +**/ + +func (client *SnatClient) ConfigureSnatContainerInterface() error { + log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.containerSnatVethName) + ip, intIpAddr, _ := net.ParseCIDR(client.localIP) + err := client.netlink.AddIPAddress(client.containerSnatVethName, ip, intIpAddr) + if err != nil { + return newErrorSnatClient(err.Error()) + } + return nil +} + +func (client *SnatClient) DeleteSnatEndpoint() error { + log.Printf("[ovs] Deleting snat veth pair %v.", client.hostSnatVethName) + err := client.netlink.DeleteLink(client.hostSnatVethName) + if err != nil { + log.Printf("[ovs] Failed to delete veth pair %v: %v.", client.hostSnatVethName, err) + return newErrorSnatClient(err.Error()) + } + + return nil +} + +func (client *SnatClient) setBridgeMac(hostPrimaryMac string) error { + hwAddr, err := net.ParseMAC(hostPrimaryMac) + if err != nil { + log.Errorf("Error while parsing host primary mac: %s error:%+v", hostPrimaryMac, err) + return err + } + + if err = client.netlink.SetLinkAddress(SnatBridgeName, hwAddr); err != nil { + log.Errorf("Error while setting macaddr on bridge: %s error:%+v", hwAddr.String(), err) + } + return err +} + +func (client *SnatClient) dropArpForSnatBridgeApipaRange(snatBridgeIP, azSnatVethIfName string) error { + var err error + _, ipCidr, _ := net.ParseCIDR(snatBridgeIP) + if err = ebtables.SetArpDropRuleForIpCidr(ipCidr.String(), azSnatVethIfName); err != nil { + log.Errorf("Error setting arp drop rule for snatbridge ip :%s", snatBridgeIP) + } + + return err +} + +/** + This function creates linux bridge which will be used for outbound connectivity by NCs +**/ +func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac string, mainInterface string) error { + _, err := net.InterfaceByName(SnatBridgeName) + if err == nil { + log.Printf("Snat Bridge already exists") + } else { + log.Printf("[net] Creating Snat bridge %v.", SnatBridgeName) + + link := netlink.BridgeLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_BRIDGE, + Name: SnatBridgeName, + }, + } + + if err := client.netlink.AddLink(&link); err != nil { + return newErrorSnatClient(err.Error()) + } + } + + log.Printf("Setting snat bridge mac: %s", hostPrimaryMac) + if err := client.setBridgeMac(hostPrimaryMac); err != nil { + return err + } + + log.Printf("Drop ARP for snat bridge ip: %s", snatBridgeIP) + if err := client.dropArpForSnatBridgeApipaRange(snatBridgeIP, azureSnatVeth0); err != nil { + return err + } + + // Create a veth pair. One end of veth will be attached to ovs bridge and other end + // of veth will be attached to linux bridge + _, err = net.InterfaceByName(azureSnatVeth0) + if err == nil { + log.Printf("Azure snat veth already exists") + return nil + } + + nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + //nolint + if err = nuc.DisableRAForInterface(SnatBridgeName); err != nil { + return err + } + + vethLink := netlink.VEthLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_VETH, + Name: azureSnatVeth0, + }, + PeerName: azureSnatVeth1, + } + + err = client.netlink.AddLink(&vethLink) + if err != nil { + log.Printf("[net] Failed to create veth pair, err:%v.", err) + return err + } + + //nolint + if err = nuc.DisableRAForInterface(azureSnatVeth0); err != nil { + return err + } + + //nolint + if err = nuc.DisableRAForInterface(azureSnatVeth1); err != nil { + return err + } + + log.Printf("Assigning %v on snat bridge", snatBridgeIP) + + ip, addr, _ := net.ParseCIDR(snatBridgeIP) + err = client.netlink.AddIPAddress(SnatBridgeName, ip, addr) + if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { + log.Printf("[net] Failed to add IP address %v: %v.", addr, err) + return newErrorSnatClient(err.Error()) + } + + if err = client.netlink.SetLinkState(SnatBridgeName, true); err != nil { + return newErrorSnatClient(err.Error()) + } + + if err = client.netlink.SetLinkState(azureSnatVeth0, true); err != nil { + return newErrorSnatClient(err.Error()) + } + + if err = client.netlink.SetLinkMaster(azureSnatVeth0, SnatBridgeName); err != nil { + return newErrorSnatClient(err.Error()) + } + + if err = client.netlink.SetLinkState(azureSnatVeth1, true); err != nil { + return newErrorSnatClient(err.Error()) + } + + if err = client.ovsctlClient.AddPortOnOVSBridge(azureSnatVeth1, mainInterface, 0); err != nil { + return newErrorSnatClient(err.Error()) + } + + return nil +} + +/** + This function adds iptable rules that will snat all traffic that has source ip in apipa range and coming via linux bridge +**/ +func (client *SnatClient) addMasqueradeRule(snatBridgeIPWithPrefix string) error { + _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) + matchCondition := fmt.Sprintf("-s %s", ipNet.String()) + return iptables.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) +} + +/** + Drop all vlan traffic on linux bridge +**/ +func (client *SnatClient) addVlanDropRule() error { + out, err := client.plClient.ExecuteCommand(l2PreroutingEntries) + if err != nil { + log.Printf("Error while listing ebtable rules %v", err) + return err + } + + out = strings.TrimSpace(out) + if strings.Contains(out, vlanDropMatch) { + log.Printf("vlan drop rule already exists") + return nil + } + + log.Printf("Adding ebtable rule to drop vlan traffic on snat bridge %v", vlanDropAddRule) + _, err = client.plClient.ExecuteCommand(vlanDropAddRule) + return err +} diff --git a/network/snat/snat_linux_test.go b/network/snat/snat_linux_test.go new file mode 100644 index 0000000000..e937e43d49 --- /dev/null +++ b/network/snat/snat_linux_test.go @@ -0,0 +1,114 @@ +package snat + +import ( + "os" + "testing" + + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/ovsctl" +) + +var anyInterface = "dummy" + +func TestMain(m *testing.M) { + exitCode := m.Run() + + // Create a dummy test network interface. + + os.Exit(exitCode) +} + +func TestAllowInboundFromHostToNC(t *testing.T) { + nl := netlink.NewNetlink() + client := &SnatClient{ + snatBridgeIP: "169.254.0.1/16", + localIP: "169.254.0.4/16", + containerSnatVethName: anyInterface, + netlink: nl, + ovsctlClient: ovsctl.NewMockOvsctl(false, "", ""), + } + + if err := nl.AddLink(&netlink.DummyLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_DUMMY, + Name: anyInterface, + }, + }); err != nil { + t.Errorf("Error adding dummy interface %v", err) + } + + if err := nl.AddLink(&netlink.DummyLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_DUMMY, + Name: SnatBridgeName, + }, + }); err != nil { + t.Errorf("Error adding dummy interface %v", err) + } + + if err := client.AllowInboundFromHostToNC(); err != nil { + t.Errorf("Error adding inbound rule: %v", err) + } + + if err := client.AllowInboundFromHostToNC(); err != nil { + t.Errorf("Error adding existing inbound rule: %v", err) + } + + if err := client.DeleteInboundFromHostToNC(); err != nil { + t.Errorf("Error removing inbound rule: %v", err) + } + + if err := nl.DeleteLink(anyInterface); err != nil { + t.Errorf("Error removing any interface link: %v", err) + } + if err := nl.DeleteLink(SnatBridgeName); err != nil { + t.Errorf("Error removing snat bridge: %v", err) + } +} + +func TestAllowInboundFromNCToHost(t *testing.T) { + nl := netlink.NewNetlink() + client := &SnatClient{ + snatBridgeIP: "169.254.0.1/16", + localIP: "169.254.0.4/16", + containerSnatVethName: anyInterface, + netlink: nl, + } + + if err := nl.AddLink(&netlink.DummyLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_DUMMY, + Name: anyInterface, + }, + }); err != nil { + t.Errorf("Error adding dummy interface %v", err) + } + + if err := nl.AddLink(&netlink.DummyLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_DUMMY, + Name: SnatBridgeName, + }, + }); err != nil { + t.Errorf("Error adding dummy interface %v", err) + } + + if err := client.AllowInboundFromNCToHost(); err != nil { + t.Errorf("Error adding inbound rule: %v", err) + } + + if err := client.AllowInboundFromNCToHost(); err != nil { + t.Errorf("Error adding existing inbound rule: %v", err) + } + + if err := client.DeleteInboundFromNCToHost(); err != nil { + t.Errorf("Error removing inbound rule: %v", err) + } + + if err := nl.DeleteLink(anyInterface); err != nil { + t.Errorf("Error removing any interface link: %v", err) + } + if err := nl.DeleteLink(SnatBridgeName); err != nil { + t.Errorf("Error removing snat bridge: %v", err) + } +} From 580bffd4fe1dd1d89df7e0fb1d34397ec68dfa03 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 15:19:03 -0700 Subject: [PATCH 45/86] Breaking, prepping for Native Snat Delete native endpoint snat route linux to remove errors and in theory, ovs should work fine again. --- network/native_endpoint_snatroute_linux.go | 114 +++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 network/native_endpoint_snatroute_linux.go diff --git a/network/native_endpoint_snatroute_linux.go b/network/native_endpoint_snatroute_linux.go new file mode 100644 index 0000000000..e36111d4ec --- /dev/null +++ b/network/native_endpoint_snatroute_linux.go @@ -0,0 +1,114 @@ +package network + +import ( + "fmt" + + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/network/snat" +) + +func NativeNewSnatClient(client *NativeEndpointClient, snatBridgeIP string, localIP string, epInfo *EndpointInfo) { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { + hostIfName := fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7]) + contIfName := fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7]) + + client.snatClient = snat.NewSnatClient(hostIfName, + contIfName, + localIP, + snatBridgeIP, + client.hostPrimaryMac, // extIf mac string + epInfo.DNS.Servers, + client.netlink, + client.ovsctlClient, // nil + client.plClient, + ) + } +} + +func NativeAddSnatEndpoint(client *NativeEndpointClient) error { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { + if err := client.snatClient.CreateSnatEndpoint(client.bridgeName); err != nil { + return err + } + } + + return nil +} + +func NativeAddSnatEndpointRules(client *NativeEndpointClient) error { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { + // Allow specific Private IPs via Snat Bridge + if err := client.snatClient.AllowIPAddressesOnSnatBridge(); err != nil { + return err + } + + // Block Private IPs via Snat Bridge + if err := client.snatClient.BlockIPAddressesOnSnatBridge(); err != nil { + return err + } + + // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge + if err := AddStaticRoute(client.netlink, client.netioshim, snat.ImdsIP, client.bridgeName); err != nil { + return err + } + + nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + if err := nuc.EnableIPForwarding(snat.SnatBridgeName); err != nil { + return err + } + + if client.allowInboundFromHostToNC { + if err := client.snatClient.AllowInboundFromHostToNC(); err != nil { + return err + } + } + + if client.allowInboundFromNCToHost { + return client.snatClient.AllowInboundFromNCToHost() + } + } + + return nil +} + +func NativeMoveSnatEndpointToContainerNS(client *NativeEndpointClient, netnsPath string, nsID uintptr) error { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { + return client.snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID) + } + + return nil +} + +func NativeSetupSnatContainerInterface(client *NativeEndpointClient) error { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { + return client.snatClient.SetupSnatContainerInterface() + } + + return nil +} + +func NativeConfigureSnatContainerInterface(client *NativeEndpointClient) error { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { + return client.snatClient.ConfigureSnatContainerInterface() + } + + return nil +} + +func NativeDeleteSnatEndpoint(client *NativeEndpointClient) error { + if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { + return client.snatClient.DeleteSnatEndpoint() + } + + return nil +} + +func NativeDeleteSnatEndpointRules(client *NativeEndpointClient) { + if client.allowInboundFromHostToNC { + client.snatClient.DeleteInboundFromHostToNC() + } + + if client.allowInboundFromNCToHost { + client.snatClient.DeleteInboundFromNCToHost() + } +} From 88a2a50f634cc1ab8f880970f40477874ab044a4 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 15:32:20 -0700 Subject: [PATCH 46/86] Go mod tidy for linting Hopefully this fixes the windows lint error --- go.mod | 2 +- go.sum | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 8acdb5db3e..f6cfe75282 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,7 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect github.com/vishvananda/netlink v1.2.1-beta.2 - github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect + github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.6.0 // indirect diff --git a/go.sum b/go.sum index 1719c6b73e..847e76e7a3 100644 --- a/go.sum +++ b/go.sum @@ -830,15 +830,12 @@ github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg= github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= From d7c159eae078fd0aacc0310adaff9497475a9541 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 15:36:49 -0700 Subject: [PATCH 47/86] Add fields to native endpoint client for snat --- network/native_endpointclient_linux.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 8edadfe8eb..8ad9af2240 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -8,6 +8,7 @@ import ( "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/network/snat" "github.com/Azure/azure-container-networking/platform" "github.com/pkg/errors" vishnetlink "github.com/vishvananda/netlink" @@ -31,13 +32,19 @@ type NativeEndpointClient struct { vnetNSName string vnetNSFileDescriptor int - nw *network - vlanID int - netnsClient NetnsInterface - netlink netlink.NetlinkInterface - netioshim netio.NetIOInterface - plClient platform.ExecClient - netUtilsClient networkutils.NetworkUtils + nw *network + snatClient snat.SnatClient + vlanID int + enableSnatOnHost bool + enableInfraVnet bool + allowInboundFromHostToNC bool + allowInboundFromNCToHost bool + enableSnatForDns bool + netnsClient NetnsInterface + netlink netlink.NetlinkInterface + netioshim netio.NetIOInterface + plClient platform.ExecClient + netUtilsClient networkutils.NetworkUtils } // Adds interfaces to the vnet (created if not existing) and vm namespace From b4c1b4178888e4e80ded75339a534bbf9df334e7 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 16:38:06 -0700 Subject: [PATCH 48/86] Using New() func to create Native Client Creation of the native endpoint client is too complicated to directly instantiate. --- network/endpoint_linux.go | 37 ++-------------------- network/native_endpointclient_linux.go | 44 +++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 9dbd21bfec..1da2fdfdb3 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -13,7 +13,6 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" "github.com/Azure/azure-container-networking/platform" @@ -92,23 +91,7 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p if vlanid != 0 { if nw.Mode == opModeNative { log.Printf("Native client") - vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, vlanid) - vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) - - epClient = &NativeEndpointClient{ - eth0VethName: nw.extIf.Name, - vlanVethName: vlanVethName, - vnetVethName: hostIfName, - containerVethName: contIfName, - vnetNSName: vnetNSName, - nw: nw, - vlanID: vlanid, - netnsClient: netns.New(), - netlink: nl, - netioshim: &netio.NetIO{}, - plClient: plc, - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - } + epClient = NewNativeEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc) } else { log.Printf("OVS client") if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { @@ -263,23 +246,7 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. epInfo := ep.getInfo() if nw.Mode == opModeNative { log.Printf("Native client") - vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, ep.VlanID) - vnetNSName := fmt.Sprintf("az_ns_%d", ep.VlanID) - - epClient = &NativeEndpointClient{ - eth0VethName: nw.extIf.Name, - vlanVethName: vlanVethName, - vnetVethName: ep.HostIfName, - containerVethName: "", - vnetNSName: vnetNSName, - nw: nw, - vlanID: ep.VlanID, - netnsClient: netns.New(), - netlink: nl, - netioshim: &netio.NetIO{}, - plClient: plc, - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - } + epClient = NewNativeEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, plc) } else { epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) } diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index b3851b85d4..3c68e89bd0 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -7,6 +7,7 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/snat" "github.com/Azure/azure-container-networking/platform" @@ -21,18 +22,20 @@ const ( ) type NativeEndpointClient struct { + bridgeName string eth0VethName string // So like eth0 vlanVethName string // So like eth0.1 vnetVethName string // Peer is containerVethName containerVethName string // Peer is vnetVethName + hostPrimaryMac net.HardwareAddr + vnetMac net.HardwareAddr containerMac net.HardwareAddr vnetNSName string vnetNSFileDescriptor int - nw *network snatClient snat.SnatClient vlanID int enableSnatOnHost bool @@ -47,6 +50,45 @@ type NativeEndpointClient struct { netUtilsClient networkutils.NetworkUtils } +func NewNativeEndpointClient( + nw *network, + ep *EndpointInfo, + vnetVethName string, + containerVethName string, + vlanid int, + localIP string, + nl netlink.NetlinkInterface, + plc platform.ExecClient) *NativeEndpointClient { + + vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, vlanid) + vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) + client := &NativeEndpointClient{ + bridgeName: nw.extIf.BridgeName, + eth0VethName: nw.extIf.Name, + vlanVethName: vlanVethName, + vnetVethName: vnetVethName, + hostPrimaryMac: nw.extIf.MacAddress, + containerVethName: containerVethName, + vnetNSName: vnetNSName, + vlanID: vlanid, + enableSnatOnHost: ep.EnableSnatOnHost, + enableInfraVnet: ep.EnableInfraVnet, + allowInboundFromHostToNC: ep.AllowInboundFromHostToNC, + allowInboundFromNCToHost: ep.AllowInboundFromNCToHost, + enableSnatForDns: ep.EnableSnatForDns, + netnsClient: netns.New(), + netlink: nl, + netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + } + + NativeNewSnatClient(client, nw.SnatBridgeIP, localIP, ep) + + return client + +} + // Adds interfaces to the vnet (created if not existing) and vm namespace func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { // VM Namespace From 257ab8c035d62e4e60c8201275aa07753716a8eb Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 16:38:19 -0700 Subject: [PATCH 49/86] Snat defaults --- network/native_endpoint_snatroute_linux.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/native_endpoint_snatroute_linux.go b/network/native_endpoint_snatroute_linux.go index e36111d4ec..a885afe684 100644 --- a/network/native_endpoint_snatroute_linux.go +++ b/network/native_endpoint_snatroute_linux.go @@ -16,10 +16,10 @@ func NativeNewSnatClient(client *NativeEndpointClient, snatBridgeIP string, loca contIfName, localIP, snatBridgeIP, - client.hostPrimaryMac, // extIf mac string + client.hostPrimaryMac.String(), epInfo.DNS.Servers, client.netlink, - client.ovsctlClient, // nil + nil, client.plClient, ) } From f6a013c2ce7e065ac51fd3ab8339e97a00b4bd52 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 16:52:15 -0700 Subject: [PATCH 50/86] Insert SNAT entry points --- network/native_endpointclient_linux.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 3c68e89bd0..ba7b838dc8 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -96,6 +96,9 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if err != nil { return err } + if err := NativeAddSnatEndpoint(client); err != nil { + return err + } // VNET Namespace return ExecuteInNS(client.vnetNSName, func() error { return client.PopulateVnet(epInfo) @@ -211,18 +214,19 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error // There are no rules to add here // Described as rules on ip addresses on the container interface - return nil + return NativeAddSnatEndpointRules(client) } func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { // Never added any endpoint rules + NativeDeleteSnatEndpointRules(client) } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { return errors.Wrap(err, "failed to move endpoint to container ns") } - return nil + return NativeMoveSnatEndpointToContainerNS(client, epInfo.NetNsPath, nsID) } func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { @@ -231,7 +235,7 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf } client.containerVethName = epInfo.IfName - return nil + return NativeSetupSnatContainerInterface(client) } // Adds routes, arp entries, etc. to the vnet and container namespaces @@ -242,6 +246,10 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return err } + if err := NativeConfigureSnatContainerInterface(client); err != nil { + return err + } + // Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes return ExecuteInNS(client.vnetNSName, func() error { return client.ConfigureVnetInterfacesAndRoutesImpl(epInfo) @@ -374,6 +382,9 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) } func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { + + NativeDeleteSnatEndpoint(client) + return ExecuteInNS(client.vnetNSName, func() error { routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) if err != nil { From 059363c98002a41d9d212b9a31f87dd5d7abd34f Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 09:28:12 -0700 Subject: [PATCH 51/86] Native Snat error handling --- network/native_endpointclient_linux.go | 29 ++++++++++++++++---------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index ba7b838dc8..8ff77b24e8 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -97,7 +97,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return err } if err := NativeAddSnatEndpoint(client); err != nil { - return err + return errors.Wrap(err, "failed to add snat endpoint") } // VNET Namespace return ExecuteInNS(client.vnetNSName, func() error { @@ -211,14 +211,13 @@ func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { } func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { - // There are no rules to add here - // Described as rules on ip addresses on the container interface - - return NativeAddSnatEndpointRules(client) + if err := NativeAddSnatEndpointRules(client); err != nil { + return errors.Wrap(err, "failed to add snat endpoint rules") + } + return nil } func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { - // Never added any endpoint rules NativeDeleteSnatEndpointRules(client) } @@ -226,7 +225,10 @@ func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointI if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { return errors.Wrap(err, "failed to move endpoint to container ns") } - return NativeMoveSnatEndpointToContainerNS(client, epInfo.NetNsPath, nsID) + if err := NativeMoveSnatEndpointToContainerNS(client, epInfo.NetNsPath, nsID); err != nil { + return errors.Wrap(err, "failed to move snat endpoint to container ns") + } + return nil } func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { @@ -235,7 +237,10 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf } client.containerVethName = epInfo.IfName - return NativeSetupSnatContainerInterface(client) + if err := NativeSetupSnatContainerInterface(client); err != nil { + return errors.Wrap(err, "failed to setup snat container interface") + } + return nil } // Adds routes, arp entries, etc. to the vnet and container namespaces @@ -246,8 +251,8 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return err } - if err := NativeConfigureSnatContainerInterface(client); err != nil { - return err + if err = NativeConfigureSnatContainerInterface(client); err != nil { + return errors.Wrap(err, "failed to configure snat container interface") } // Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes @@ -383,7 +388,9 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { - NativeDeleteSnatEndpoint(client) + if err := NativeDeleteSnatEndpoint(client); err != nil { + return errors.Wrap(err, "failed to delete snat endpoint") + } return ExecuteInNS(client.vnetNSName, func() error { routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) From 6c390d30c1d46bdeab535469dfd137a1c1a01cb5 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 09:41:36 -0700 Subject: [PATCH 52/86] Breaking, decouple ovsctl from snat Proposed Solution implementation Moved ovsctlClient.AddPortOnOVSBridge to ovs_endpoint_snatroute_linux.go. Removed ovsctlclient from NewSnatClient. Removed ovsctlClient from testing file. --- network/native_endpoint_snatroute_linux.go | 1 - network/ovs_endpoint_snatroute_linux.go | 5 ++++- network/snat/snat_linux.go | 23 +++++++++------------- network/snat/snat_linux_test.go | 2 -- 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/network/native_endpoint_snatroute_linux.go b/network/native_endpoint_snatroute_linux.go index a885afe684..134c9e1781 100644 --- a/network/native_endpoint_snatroute_linux.go +++ b/network/native_endpoint_snatroute_linux.go @@ -19,7 +19,6 @@ func NativeNewSnatClient(client *NativeEndpointClient, snatBridgeIP string, loca client.hostPrimaryMac.String(), epInfo.DNS.Servers, client.netlink, - nil, client.plClient, ) } diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index 5dbe1b9499..f98c9f4d4d 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -19,7 +19,6 @@ func NewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP strin client.hostPrimaryMac, epInfo.DNS.Servers, client.netlink, - client.ovsctlClient, client.plClient, ) } @@ -30,6 +29,10 @@ func AddSnatEndpoint(client *OVSEndpointClient) error { if err := client.snatClient.CreateSnatEndpoint(client.bridgeName); err != nil { return err } + + if err := client.ovsctlClient.AddPortOnOVSBridge(snat.AzureSnatVeth1, client.bridgeName, 0); err != nil { + return err + } } return nil diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go index a8b3ba0fbf..5774f0e5ab 100644 --- a/network/snat/snat_linux.go +++ b/network/snat/snat_linux.go @@ -14,13 +14,12 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" - "github.com/Azure/azure-container-networking/ovsctl" "github.com/Azure/azure-container-networking/platform" ) const ( azureSnatVeth0 = "azSnatveth0" - azureSnatVeth1 = "azSnatveth1" + AzureSnatVeth1 = "azSnatveth1" azureSnatIfName = "eth1" SnatBridgeName = "azSnatbr" ImdsIP = "169.254.169.254/32" @@ -44,8 +43,8 @@ type SnatClient struct { snatBridgeIP string SkipAddressesFromBlock []string netlink netlink.NetlinkInterface - ovsctlClient ovsctl.OvsInterface - plClient platform.ExecClient + + plClient platform.ExecClient } func NewSnatClient(hostIfName string, @@ -55,7 +54,7 @@ func NewSnatClient(hostIfName string, hostPrimaryMac string, skipAddressesFromBlock []string, nl netlink.NetlinkInterface, - ovsctlClient ovsctl.OvsInterface, + plClient platform.ExecClient, ) SnatClient { log.Printf("Initialize new snat client") @@ -66,8 +65,8 @@ func NewSnatClient(hostIfName string, snatBridgeIP: snatBridgeIP, hostPrimaryMac: hostPrimaryMac, netlink: nl, - ovsctlClient: ovsctlClient, - plClient: plClient, + + plClient: plClient, } snatClient.SkipAddressesFromBlock = append(snatClient.SkipAddressesFromBlock, skipAddressesFromBlock...) @@ -418,7 +417,7 @@ func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac s Type: netlink.LINK_TYPE_VETH, Name: azureSnatVeth0, }, - PeerName: azureSnatVeth1, + PeerName: AzureSnatVeth1, } err = client.netlink.AddLink(&vethLink) @@ -433,7 +432,7 @@ func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac s } //nolint - if err = nuc.DisableRAForInterface(azureSnatVeth1); err != nil { + if err = nuc.DisableRAForInterface(AzureSnatVeth1); err != nil { return err } @@ -458,11 +457,7 @@ func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac s return newErrorSnatClient(err.Error()) } - if err = client.netlink.SetLinkState(azureSnatVeth1, true); err != nil { - return newErrorSnatClient(err.Error()) - } - - if err = client.ovsctlClient.AddPortOnOVSBridge(azureSnatVeth1, mainInterface, 0); err != nil { + if err = client.netlink.SetLinkState(AzureSnatVeth1, true); err != nil { return newErrorSnatClient(err.Error()) } diff --git a/network/snat/snat_linux_test.go b/network/snat/snat_linux_test.go index e937e43d49..69fe80bb5e 100644 --- a/network/snat/snat_linux_test.go +++ b/network/snat/snat_linux_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/ovsctl" ) var anyInterface = "dummy" @@ -25,7 +24,6 @@ func TestAllowInboundFromHostToNC(t *testing.T) { localIP: "169.254.0.4/16", containerSnatVethName: anyInterface, netlink: nl, - ovsctlClient: ovsctl.NewMockOvsctl(false, "", ""), } if err := nl.AddLink(&netlink.DummyLink{ From 9cb6a51e3f4a008fe09c4a2e5e8a158dc99c7a36 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 10:48:25 -0700 Subject: [PATCH 53/86] Delete unecessary ovssnat files --- network/ovssnat/ovssnat_linux.go | 500 -------------------------- network/ovssnat/ovssnat_linux_test.go | 114 ------ 2 files changed, 614 deletions(-) delete mode 100644 network/ovssnat/ovssnat_linux.go delete mode 100644 network/ovssnat/ovssnat_linux_test.go diff --git a/network/ovssnat/ovssnat_linux.go b/network/ovssnat/ovssnat_linux.go deleted file mode 100644 index f053de1ed1..0000000000 --- a/network/ovssnat/ovssnat_linux.go +++ /dev/null @@ -1,500 +0,0 @@ -// Copyright 2017 Microsoft. All rights reserved. -// MIT License - -package ovssnat - -import ( - "errors" - "fmt" - "net" - "strings" - - "github.com/Azure/azure-container-networking/ebtables" - "github.com/Azure/azure-container-networking/iptables" - "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network/networkutils" - "github.com/Azure/azure-container-networking/ovsctl" - "github.com/Azure/azure-container-networking/platform" -) - -const ( - azureSnatVeth0 = "azSnatveth0" - azureSnatVeth1 = "azSnatveth1" - azureSnatIfName = "eth1" - SnatBridgeName = "azSnatbr" - ImdsIP = "169.254.169.254/32" - vlanDropDeleteRule = "ebtables -t nat -D PREROUTING -p 802_1Q -j DROP" - vlanDropAddRule = "ebtables -t nat -A PREROUTING -p 802_1Q -j DROP" - vlanDropMatch = "-p 802_1Q -j DROP" - l2PreroutingEntries = "ebtables -t nat -L PREROUTING" -) - -var errorSnatClient = errors.New("OVSSnatClient Error") - -func newErrorSnatClient(errStr string) error { - return fmt.Errorf("%w : %s", errorSnatClient, errStr) -} - -type SnatClient struct { - hostSnatVethName string - hostPrimaryMac string - containerSnatVethName string - localIP string - snatBridgeIP string - SkipAddressesFromBlock []string - netlink netlink.NetlinkInterface - ovsctlClient ovsctl.OvsInterface - plClient platform.ExecClient -} - -func NewSnatClient(hostIfName string, - contIfName string, - localIP string, - snatBridgeIP string, - hostPrimaryMac string, - skipAddressesFromBlock []string, - nl netlink.NetlinkInterface, - ovsctlClient ovsctl.OvsInterface, - plClient platform.ExecClient, -) SnatClient { - log.Printf("Initialize new snat client") - snatClient := SnatClient{ - hostSnatVethName: hostIfName, - containerSnatVethName: contIfName, - localIP: localIP, - snatBridgeIP: snatBridgeIP, - hostPrimaryMac: hostPrimaryMac, - netlink: nl, - ovsctlClient: ovsctlClient, - plClient: plClient, - } - - snatClient.SkipAddressesFromBlock = append(snatClient.SkipAddressesFromBlock, skipAddressesFromBlock...) - - log.Printf("Initialize new snat client %+v", snatClient) - - return snatClient -} - -func (client *SnatClient) CreateSnatEndpoint(bridgeName string) error { - // Create linux Bridge for outbound connectivity - if err := client.createSnatBridge(client.snatBridgeIP, client.hostPrimaryMac, bridgeName); err != nil { - log.Printf("creating snat bridge failed with error %v", err) - return err - } - - // SNAT Rule to masquerade packets destined to non-vnet ip - if err := client.addMasqueradeRule(client.snatBridgeIP); err != nil { - log.Printf("Adding snat rule failed with error %v", err) - return err - } - - // Drop all vlan packets coming via linux bridge. - if err := client.addVlanDropRule(); err != nil { - log.Printf("Adding vlan drop rule failed with error %v", err) - return err - } - - epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) - // Create veth pair to tie one end to container and other end to linux bridge - if err := epc.CreateEndpoint(client.hostSnatVethName, client.containerSnatVethName); err != nil { - log.Printf("Creating Snat Endpoint failed with error %v", err) - return newErrorSnatClient(err.Error()) - } - - err := client.netlink.SetLinkMaster(client.hostSnatVethName, SnatBridgeName) - if err != nil { - return newErrorSnatClient(err.Error()) - } - return nil -} - -// AllowIPAddressesOnSnatBridge adds iptables rules that allows only specific Private IPs via linux bridge -func (client *SnatClient) AllowIPAddressesOnSnatBridge() error { - if err := networkutils.AllowIPAddresses(SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { - log.Printf("AllowIPAddresses failed with error %v", err) - return newErrorSnatClient(err.Error()) - } - - return nil -} - -// BlockIPAddressesOnSnatBridge adds iptables rules that blocks all private IPs flowing via linux bridge -func (client *SnatClient) BlockIPAddressesOnSnatBridge() error { - if err := networkutils.BlockIPAddresses(SnatBridgeName, iptables.Append); err != nil { - log.Printf("AllowIPAddresses failed with error %v", err) - return newErrorSnatClient(err.Error()) - } - - return nil -} - -/** - Move container veth inside container network namespace -**/ -func (client *SnatClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { - log.Printf("[ovs] Setting link %v netns %v.", client.containerSnatVethName, netnsPath) - err := client.netlink.SetLinkNetNs(client.containerSnatVethName, nsID) - if err != nil { - return newErrorSnatClient(err.Error()) - } - return nil -} - -/** - Configure Routes and setup name for container veth -**/ -func (client *SnatClient) SetupSnatContainerInterface() error { - epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) - if err := epc.SetupContainerInterface(client.containerSnatVethName, azureSnatIfName); err != nil { - return newErrorSnatClient(err.Error()) - } - - client.containerSnatVethName = azureSnatIfName - - return nil -} - -func getNCLocalAndGatewayIP(client *SnatClient) (net.IP, net.IP) { - bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) - containerIP, _, _ := net.ParseCIDR(client.localIP) - return bridgeIP, containerIP -} - -/** - This function adds iptables rules that allows only host to NC communication and not the other way -**/ -func (client *SnatClient) AllowInboundFromHostToNC() error { - bridgeIP, containerIP := getNCLocalAndGatewayIP(client) - - // Create CNI Ouptut chain - if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { - log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) - return newErrorSnatClient(err.Error()) - } - - // Forward traffic from Ouptut chain to CNI Output chain - if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { - log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err) - return newErrorSnatClient(err.Error()) - } - - // Allow connection from Host to NC - matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) - err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) - if err != nil { - log.Printf("AllowInboundFromHostToNC: Inserting output rule failed: %v", err) - return newErrorSnatClient(err.Error()) - } - - // Create cniinput chain - if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { - log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err) - return newErrorSnatClient(err.Error()) - } - - // Forward from Input to cniinput chain - if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { - log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIInputChain, err) - return newErrorSnatClient(err.Error()) - } - - // Accept packets from NC only if established connection - matchCondition = fmt.Sprintf(" -i %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related) - err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) - if err != nil { - log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err) - return newErrorSnatClient(err.Error()) - } - - snatContainerVeth, _ := net.InterfaceByName(client.containerSnatVethName) - - // Add static arp entry for localIP to prevent arp going out of VM - log.Printf("Adding static arp entry for ip %s mac %s", containerIP, snatContainerVeth.HardwareAddr.String()) - err = client.netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr, false) - if err != nil { - log.Printf("AllowInboundFromHostToNC: Error adding static arp entry for ip %s mac %s: %v", containerIP, snatContainerVeth.HardwareAddr.String(), err) - return newErrorSnatClient(err.Error()) - } - - return nil -} - -func (client *SnatClient) DeleteInboundFromHostToNC() error { - bridgeIP, containerIP := getNCLocalAndGatewayIP(client) - - // Delete allow connection from Host to NC - matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String()) - err := iptables.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) - if err != nil { - log.Printf("DeleteInboundFromHostToNC: Error removing output rule %v", err) - } - - // Remove static arp entry added for container local IP - log.Printf("Removing static arp entry for ip %s ", containerIP) - err = client.netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil, false) - if err != nil { - log.Printf("AllowInboundFromHostToNC: Error removing static arp entry for ip %s: %v", containerIP, err) - } - - return err -} - -/** - This function adds iptables rules that allows only NC to Host communication and not the other way -**/ -func (client *SnatClient) AllowInboundFromNCToHost() error { - bridgeIP, containerIP := getNCLocalAndGatewayIP(client) - - // Create CNI Input chain - if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil { - log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err) - return err - } - - // Forward traffic from Input to cniinput chain - if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil { - log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIInputChain, err) - return err - } - - // Allow NC to Host connection - matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) - err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) - if err != nil { - log.Printf("AllowInboundFromHostToNC: Inserting output rule failed: %v", err) - return err - } - - // Create CNI output chain - if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { - log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) - return err - } - - // Forward traffic from Output to CNI Output chain - if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil { - log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err) - return err - } - - // Accept packets from Host only if established connection - matchCondition = fmt.Sprintf(" -o %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related) - err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept) - if err != nil { - log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err) - return err - } - - snatContainerVeth, _ := net.InterfaceByName(client.containerSnatVethName) - - // Add static arp entry for localIP to prevent arp going out of VM - log.Printf("Adding static arp entry for ip %s mac %s", containerIP, snatContainerVeth.HardwareAddr.String()) - err = client.netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr, false) - if err != nil { - log.Printf("AllowInboundFromNCToHost: Error adding static arp entry for ip %s mac %s: %v", containerIP, snatContainerVeth.HardwareAddr.String(), err) - } - - return err -} - -func (client *SnatClient) DeleteInboundFromNCToHost() error { - bridgeIP, containerIP := getNCLocalAndGatewayIP(client) - - // Delete allow NC to Host connection - matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String()) - err := iptables.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept) - if err != nil { - log.Printf("DeleteInboundFromNCToHost: Error removing output rule %v", err) - } - - // Remove static arp entry added for container local IP - log.Printf("Removing static arp entry for ip %s ", containerIP) - err = client.netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil, false) - if err != nil { - log.Printf("DeleteInboundFromNCToHost: Error removing static arp entry for ip %s: %v", containerIP, err) - } - - return err -} - -/** - Configures Local IP Address for container Veth -**/ - -func (client *SnatClient) ConfigureSnatContainerInterface() error { - log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.containerSnatVethName) - ip, intIpAddr, _ := net.ParseCIDR(client.localIP) - err := client.netlink.AddIPAddress(client.containerSnatVethName, ip, intIpAddr) - if err != nil { - return newErrorSnatClient(err.Error()) - } - return nil -} - -func (client *SnatClient) DeleteSnatEndpoint() error { - log.Printf("[ovs] Deleting snat veth pair %v.", client.hostSnatVethName) - err := client.netlink.DeleteLink(client.hostSnatVethName) - if err != nil { - log.Printf("[ovs] Failed to delete veth pair %v: %v.", client.hostSnatVethName, err) - return newErrorSnatClient(err.Error()) - } - - return nil -} - -func (client *SnatClient) setBridgeMac(hostPrimaryMac string) error { - hwAddr, err := net.ParseMAC(hostPrimaryMac) - if err != nil { - log.Errorf("Error while parsing host primary mac: %s error:%+v", hostPrimaryMac, err) - return err - } - - if err = client.netlink.SetLinkAddress(SnatBridgeName, hwAddr); err != nil { - log.Errorf("Error while setting macaddr on bridge: %s error:%+v", hwAddr.String(), err) - } - return err -} - -func (client *SnatClient) dropArpForSnatBridgeApipaRange(snatBridgeIP, azSnatVethIfName string) error { - var err error - _, ipCidr, _ := net.ParseCIDR(snatBridgeIP) - if err = ebtables.SetArpDropRuleForIpCidr(ipCidr.String(), azSnatVethIfName); err != nil { - log.Errorf("Error setting arp drop rule for snatbridge ip :%s", snatBridgeIP) - } - - return err -} - -/** - This function creates linux bridge which will be used for outbound connectivity by NCs -**/ -func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac string, mainInterface string) error { - _, err := net.InterfaceByName(SnatBridgeName) - if err == nil { - log.Printf("Snat Bridge already exists") - } else { - log.Printf("[net] Creating Snat bridge %v.", SnatBridgeName) - - link := netlink.BridgeLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_BRIDGE, - Name: SnatBridgeName, - }, - } - - if err := client.netlink.AddLink(&link); err != nil { - return newErrorSnatClient(err.Error()) - } - } - - log.Printf("Setting snat bridge mac: %s", hostPrimaryMac) - if err := client.setBridgeMac(hostPrimaryMac); err != nil { - return err - } - - log.Printf("Drop ARP for snat bridge ip: %s", snatBridgeIP) - if err := client.dropArpForSnatBridgeApipaRange(snatBridgeIP, azureSnatVeth0); err != nil { - return err - } - - // Create a veth pair. One end of veth will be attached to ovs bridge and other end - // of veth will be attached to linux bridge - _, err = net.InterfaceByName(azureSnatVeth0) - if err == nil { - log.Printf("Azure snat veth already exists") - return nil - } - - nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) - //nolint - if err = nuc.DisableRAForInterface(SnatBridgeName); err != nil { - return err - } - - vethLink := netlink.VEthLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_VETH, - Name: azureSnatVeth0, - }, - PeerName: azureSnatVeth1, - } - - err = client.netlink.AddLink(&vethLink) - if err != nil { - log.Printf("[net] Failed to create veth pair, err:%v.", err) - return err - } - - //nolint - if err = nuc.DisableRAForInterface(azureSnatVeth0); err != nil { - return err - } - - //nolint - if err = nuc.DisableRAForInterface(azureSnatVeth1); err != nil { - return err - } - - log.Printf("Assigning %v on snat bridge", snatBridgeIP) - - ip, addr, _ := net.ParseCIDR(snatBridgeIP) - err = client.netlink.AddIPAddress(SnatBridgeName, ip, addr) - if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { - log.Printf("[net] Failed to add IP address %v: %v.", addr, err) - return newErrorSnatClient(err.Error()) - } - - if err = client.netlink.SetLinkState(SnatBridgeName, true); err != nil { - return newErrorSnatClient(err.Error()) - } - - if err = client.netlink.SetLinkState(azureSnatVeth0, true); err != nil { - return newErrorSnatClient(err.Error()) - } - - if err = client.netlink.SetLinkMaster(azureSnatVeth0, SnatBridgeName); err != nil { - return newErrorSnatClient(err.Error()) - } - - if err = client.netlink.SetLinkState(azureSnatVeth1, true); err != nil { - return newErrorSnatClient(err.Error()) - } - - if err = client.ovsctlClient.AddPortOnOVSBridge(azureSnatVeth1, mainInterface, 0); err != nil { - return newErrorSnatClient(err.Error()) - } - - return nil -} - -/** - This function adds iptable rules that will snat all traffic that has source ip in apipa range and coming via linux bridge -**/ -func (client *SnatClient) addMasqueradeRule(snatBridgeIPWithPrefix string) error { - _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) - matchCondition := fmt.Sprintf("-s %s", ipNet.String()) - return iptables.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) -} - -/** - Drop all vlan traffic on linux bridge -**/ -func (client *SnatClient) addVlanDropRule() error { - out, err := client.plClient.ExecuteCommand(l2PreroutingEntries) - if err != nil { - log.Printf("Error while listing ebtable rules %v", err) - return err - } - - out = strings.TrimSpace(out) - if strings.Contains(out, vlanDropMatch) { - log.Printf("vlan drop rule already exists") - return nil - } - - log.Printf("Adding ebtable rule to drop vlan traffic on snat bridge %v", vlanDropAddRule) - _, err = client.plClient.ExecuteCommand(vlanDropAddRule) - return err -} diff --git a/network/ovssnat/ovssnat_linux_test.go b/network/ovssnat/ovssnat_linux_test.go deleted file mode 100644 index c7e6879da4..0000000000 --- a/network/ovssnat/ovssnat_linux_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package ovssnat - -import ( - "os" - "testing" - - "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/ovsctl" -) - -var anyInterface = "dummy" - -func TestMain(m *testing.M) { - exitCode := m.Run() - - // Create a dummy test network interface. - - os.Exit(exitCode) -} - -func TestAllowInboundFromHostToNC(t *testing.T) { - nl := netlink.NewNetlink() - client := &SnatClient{ - snatBridgeIP: "169.254.0.1/16", - localIP: "169.254.0.4/16", - containerSnatVethName: anyInterface, - netlink: nl, - ovsctlClient: ovsctl.NewMockOvsctl(false, "", ""), - } - - if err := nl.AddLink(&netlink.DummyLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_DUMMY, - Name: anyInterface, - }, - }); err != nil { - t.Errorf("Error adding dummy interface %v", err) - } - - if err := nl.AddLink(&netlink.DummyLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_DUMMY, - Name: SnatBridgeName, - }, - }); err != nil { - t.Errorf("Error adding dummy interface %v", err) - } - - if err := client.AllowInboundFromHostToNC(); err != nil { - t.Errorf("Error adding inbound rule: %v", err) - } - - if err := client.AllowInboundFromHostToNC(); err != nil { - t.Errorf("Error adding existing inbound rule: %v", err) - } - - if err := client.DeleteInboundFromHostToNC(); err != nil { - t.Errorf("Error removing inbound rule: %v", err) - } - - if err := nl.DeleteLink(anyInterface); err != nil { - t.Errorf("Error removing any interface link: %v", err) - } - if err := nl.DeleteLink(SnatBridgeName); err != nil { - t.Errorf("Error removing snat bridge: %v", err) - } -} - -func TestAllowInboundFromNCToHost(t *testing.T) { - nl := netlink.NewNetlink() - client := &SnatClient{ - snatBridgeIP: "169.254.0.1/16", - localIP: "169.254.0.4/16", - containerSnatVethName: anyInterface, - netlink: nl, - } - - if err := nl.AddLink(&netlink.DummyLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_DUMMY, - Name: anyInterface, - }, - }); err != nil { - t.Errorf("Error adding dummy interface %v", err) - } - - if err := nl.AddLink(&netlink.DummyLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_DUMMY, - Name: SnatBridgeName, - }, - }); err != nil { - t.Errorf("Error adding dummy interface %v", err) - } - - if err := client.AllowInboundFromNCToHost(); err != nil { - t.Errorf("Error adding inbound rule: %v", err) - } - - if err := client.AllowInboundFromNCToHost(); err != nil { - t.Errorf("Error adding existing inbound rule: %v", err) - } - - if err := client.DeleteInboundFromNCToHost(); err != nil { - t.Errorf("Error removing inbound rule: %v", err) - } - - if err := nl.DeleteLink(anyInterface); err != nil { - t.Errorf("Error removing any interface link: %v", err) - } - if err := nl.DeleteLink(SnatBridgeName); err != nil { - t.Errorf("Error removing snat bridge: %v", err) - } -} From 9c150bd1d9c43a45844c8a90f68e9c9923ae23db Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 10:56:21 -0700 Subject: [PATCH 54/86] No lint on vishvananda netns Maybe this will fix the windows linter? --- netns/netns.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/netns/netns.go b/netns/netns.go index 7937d02317..c5fc351e2b 100644 --- a/netns/netns.go +++ b/netns/netns.go @@ -1,7 +1,9 @@ +//nolint package netns import ( "github.com/pkg/errors" + //nolint "github.com/vishvananda/netns" ) From ef4b49fb1a8653566c9e6516708fbec9e6f3686b Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 11:24:59 -0700 Subject: [PATCH 55/86] Build linux only for netns package Maybe this fixes the linter error? --- netns/mocknetns.go | 3 +++ netns/netns.go | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index f9a00821f9..432fe03258 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -1,3 +1,6 @@ +//go:build linux +// +build linux + package netns import ( diff --git a/netns/netns.go b/netns/netns.go index c5fc351e2b..11a48a4be1 100644 --- a/netns/netns.go +++ b/netns/netns.go @@ -1,9 +1,11 @@ +//go:build linux +// +build linux + //nolint package netns import ( "github.com/pkg/errors" - //nolint "github.com/vishvananda/netns" ) From df83adac306b87a183cd2fe41412dca05881b52f Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 11:34:43 -0700 Subject: [PATCH 56/86] Remove nolint to see if linter fails --- netns/netns.go | 1 - 1 file changed, 1 deletion(-) diff --git a/netns/netns.go b/netns/netns.go index 11a48a4be1..dba8f30f80 100644 --- a/netns/netns.go +++ b/netns/netns.go @@ -1,7 +1,6 @@ //go:build linux // +build linux -//nolint package netns import ( From aa5baed45d0a1cfc45082d926d7cca2e3652115e Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 16:51:15 -0700 Subject: [PATCH 57/86] Breaking, removed bridgeName bridgeName refers to the OVS Switch I believe --- network/snat/snat_linux.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go index 5774f0e5ab..48cdd8d7e6 100644 --- a/network/snat/snat_linux.go +++ b/network/snat/snat_linux.go @@ -76,9 +76,9 @@ func NewSnatClient(hostIfName string, return snatClient } -func (client *SnatClient) CreateSnatEndpoint(bridgeName string) error { +func (client *SnatClient) CreateSnatEndpoint() error { // Create linux Bridge for outbound connectivity - if err := client.createSnatBridge(client.snatBridgeIP, client.hostPrimaryMac, bridgeName); err != nil { + if err := client.createSnatBridge(client.snatBridgeIP, client.hostPrimaryMac); err != nil { log.Printf("creating snat bridge failed with error %v", err) return err } @@ -369,7 +369,7 @@ func (client *SnatClient) dropArpForSnatBridgeApipaRange(snatBridgeIP, azSnatVet /** This function creates linux bridge which will be used for outbound connectivity by NCs **/ -func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac string, mainInterface string) error { +func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac string) error { _, err := net.InterfaceByName(SnatBridgeName) if err == nil { log.Printf("Snat Bridge already exists") From 6a1a06c1a465b300f3136ba4538df41ce0ee3470 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 16:52:21 -0700 Subject: [PATCH 58/86] If native uses snat bridge, should also get IP --- network/endpoint_linux.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 1da2fdfdb3..48372aa4e5 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -91,6 +91,9 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p if vlanid != 0 { if nw.Mode == opModeNative { log.Printf("Native client") + if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { + nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) + } epClient = NewNativeEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc) } else { log.Printf("OVS client") From f3bc95b92bcf63becb696a01e98db429866caacf Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 16:53:14 -0700 Subject: [PATCH 59/86] Breaking, Decouple or Wrap snat route --- network/endpoint_snatroute_linux.go | 74 ++++++++++++++++++ network/native_endpoint_snatroute_linux.go | 75 +++++------------- network/ovs_endpoint_snatroute_linux.go | 91 +++++++--------------- 3 files changed, 123 insertions(+), 117 deletions(-) create mode 100644 network/endpoint_snatroute_linux.go diff --git a/network/endpoint_snatroute_linux.go b/network/endpoint_snatroute_linux.go new file mode 100644 index 0000000000..7958b833ba --- /dev/null +++ b/network/endpoint_snatroute_linux.go @@ -0,0 +1,74 @@ +package network + +import ( + "fmt" + + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/network/snat" + "github.com/Azure/azure-container-networking/platform" +) + +func GetSnatHostIfName(epInfo *EndpointInfo) string { + return fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7]) + +} +func GetSnatContIfName(epInfo *EndpointInfo) string { + return fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7]) +} +func AddSnatEndpoint(snatClient snat.SnatClient) error { + return snatClient.CreateSnatEndpoint() +} + +func AddSnatEndpointRules(snatClient snat.SnatClient, hostToNC, NCToHost bool, nl netlink.NetlinkInterface, plc platform.ExecClient) error { + // Allow specific Private IPs via Snat Bridge + if err := snatClient.AllowIPAddressesOnSnatBridge(); err != nil { + return err + } + + // Block Private IPs via Snat Bridge + if err := snatClient.BlockIPAddressesOnSnatBridge(); err != nil { + return err + } + nuc := networkutils.NewNetworkUtils(nl, plc) + if err := nuc.EnableIPForwarding(snat.SnatBridgeName); err != nil { + return err + } + + if hostToNC { + if err := snatClient.AllowInboundFromHostToNC(); err != nil { + return err + } + } + + if NCToHost { + return snatClient.AllowInboundFromNCToHost() + } + return nil +} + +func MoveSnatEndpointToContainerNS(snatClient snat.SnatClient, netnsPath string, nsID uintptr) error { + return snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID) +} + +func SetupSnatContainerInterface(snatClient snat.SnatClient) error { + return snatClient.SetupSnatContainerInterface() +} + +func ConfigureSnatContainerInterface(snatClient snat.SnatClient) error { + return snatClient.ConfigureSnatContainerInterface() +} + +func DeleteSnatEndpoint(snatClient snat.SnatClient) error { + return snatClient.DeleteSnatEndpoint() +} + +func DeleteSnatEndpointRules(snatClient snat.SnatClient, hostToNC, NCToHost bool) { + if hostToNC { + snatClient.DeleteInboundFromHostToNC() + } + + if NCToHost { + snatClient.DeleteInboundFromNCToHost() + } +} diff --git a/network/native_endpoint_snatroute_linux.go b/network/native_endpoint_snatroute_linux.go index 134c9e1781..3c66c2f483 100644 --- a/network/native_endpoint_snatroute_linux.go +++ b/network/native_endpoint_snatroute_linux.go @@ -1,19 +1,17 @@ package network import ( - "fmt" - - "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/snat" ) +func nativeEnableSnat(client *NativeEndpointClient) bool { + return client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns +} func NativeNewSnatClient(client *NativeEndpointClient, snatBridgeIP string, localIP string, epInfo *EndpointInfo) { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - hostIfName := fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7]) - contIfName := fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7]) - - client.snatClient = snat.NewSnatClient(hostIfName, - contIfName, + if nativeEnableSnat(client) { + client.snatClient = snat.NewSnatClient( + GetSnatHostIfName(epInfo), + GetSnatContIfName(epInfo), localIP, snatBridgeIP, client.hostPrimaryMac.String(), @@ -23,91 +21,58 @@ func NativeNewSnatClient(client *NativeEndpointClient, snatBridgeIP string, loca ) } } - func NativeAddSnatEndpoint(client *NativeEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - if err := client.snatClient.CreateSnatEndpoint(client.bridgeName); err != nil { + if nativeEnableSnat(client) { + if err := AddSnatEndpoint(client.snatClient); err != nil { return err } } - return nil } func NativeAddSnatEndpointRules(client *NativeEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - // Allow specific Private IPs via Snat Bridge - if err := client.snatClient.AllowIPAddressesOnSnatBridge(); err != nil { - return err - } - - // Block Private IPs via Snat Bridge - if err := client.snatClient.BlockIPAddressesOnSnatBridge(); err != nil { - return err - } - + if nativeEnableSnat(client) { // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge - if err := AddStaticRoute(client.netlink, client.netioshim, snat.ImdsIP, client.bridgeName); err != nil { + if err := AddSnatEndpointRules(client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.netlink, client.plClient); err != nil { return err } - - nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) - if err := nuc.EnableIPForwarding(snat.SnatBridgeName); err != nil { - return err - } - - if client.allowInboundFromHostToNC { - if err := client.snatClient.AllowInboundFromHostToNC(); err != nil { - return err - } - } - - if client.allowInboundFromNCToHost { - return client.snatClient.AllowInboundFromNCToHost() - } } return nil } func NativeMoveSnatEndpointToContainerNS(client *NativeEndpointClient, netnsPath string, nsID uintptr) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - return client.snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID) + if nativeEnableSnat(client) { + return MoveSnatEndpointToContainerNS(client.snatClient, netnsPath, nsID) } return nil } func NativeSetupSnatContainerInterface(client *NativeEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - return client.snatClient.SetupSnatContainerInterface() + if nativeEnableSnat(client) { + return SetupSnatContainerInterface(client.snatClient) } return nil } func NativeConfigureSnatContainerInterface(client *NativeEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - return client.snatClient.ConfigureSnatContainerInterface() + if nativeEnableSnat(client) { + return ConfigureSnatContainerInterface(client.snatClient) } return nil } func NativeDeleteSnatEndpoint(client *NativeEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - return client.snatClient.DeleteSnatEndpoint() + if nativeEnableSnat(client) { + return DeleteSnatEndpoint(client.snatClient) } return nil } func NativeDeleteSnatEndpointRules(client *NativeEndpointClient) { - if client.allowInboundFromHostToNC { - client.snatClient.DeleteInboundFromHostToNC() - } - - if client.allowInboundFromNCToHost { - client.snatClient.DeleteInboundFromNCToHost() - } + DeleteSnatEndpointRules(client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost) } diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index f98c9f4d4d..a42f28b74c 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -1,19 +1,17 @@ package network import ( - "fmt" - - "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/snat" ) -func NewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP string, epInfo *EndpointInfo) { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - hostIfName := fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7]) - contIfName := fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7]) - - client.snatClient = snat.NewSnatClient(hostIfName, - contIfName, +func ovsEnableSnat(client *OVSEndpointClient) bool { + return client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns +} +func OVSNewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP string, epInfo *EndpointInfo) { + if ovsEnableSnat(client) { + client.snatClient = snat.NewSnatClient( + GetSnatHostIfName(epInfo), + GetSnatContIfName(epInfo), localIP, snatBridgeIP, client.hostPrimaryMac, @@ -23,95 +21,64 @@ func NewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP strin ) } } - -func AddSnatEndpoint(client *OVSEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - if err := client.snatClient.CreateSnatEndpoint(client.bridgeName); err != nil { +func OVSAddSnatEndpoint(client *OVSEndpointClient) error { + if ovsEnableSnat(client) { + if err := AddSnatEndpoint(client.snatClient); err != nil { return err } - if err := client.ovsctlClient.AddPortOnOVSBridge(snat.AzureSnatVeth1, client.bridgeName, 0); err != nil { return err } } - return nil } -func AddSnatEndpointRules(client *OVSEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - // Allow specific Private IPs via Snat Bridge - if err := client.snatClient.AllowIPAddressesOnSnatBridge(); err != nil { - return err - } - - // Block Private IPs via Snat Bridge - if err := client.snatClient.BlockIPAddressesOnSnatBridge(); err != nil { - return err - } - +func OVSAddSnatEndpointRules(client *OVSEndpointClient) error { + if ovsEnableSnat(client) { // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge - if err := AddStaticRoute(client.netlink, client.netioshim, snat.ImdsIP, client.bridgeName); err != nil { + if err := AddSnatEndpointRules(client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.netlink, client.plClient); err != nil { return err } - - nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) - if err := nuc.EnableIPForwarding(snat.SnatBridgeName); err != nil { + if err := AddStaticRoute(client.netlink, client.netioshim, snat.ImdsIP, client.bridgeName); err != nil { return err } - - if client.allowInboundFromHostToNC { - if err := client.snatClient.AllowInboundFromHostToNC(); err != nil { - return err - } - } - - if client.allowInboundFromNCToHost { - return client.snatClient.AllowInboundFromNCToHost() - } } return nil } -func MoveSnatEndpointToContainerNS(client *OVSEndpointClient, netnsPath string, nsID uintptr) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - return client.snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID) +func OVSMoveSnatEndpointToContainerNS(client *OVSEndpointClient, netnsPath string, nsID uintptr) error { + if ovsEnableSnat(client) { + return MoveSnatEndpointToContainerNS(client.snatClient, netnsPath, nsID) } return nil } -func SetupSnatContainerInterface(client *OVSEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - return client.snatClient.SetupSnatContainerInterface() +func OVSSetupSnatContainerInterface(client *OVSEndpointClient) error { + if ovsEnableSnat(client) { + return SetupSnatContainerInterface(client.snatClient) } return nil } -func ConfigureSnatContainerInterface(client *OVSEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - return client.snatClient.ConfigureSnatContainerInterface() +func OVSConfigureSnatContainerInterface(client *OVSEndpointClient) error { + if ovsEnableSnat(client) { + return ConfigureSnatContainerInterface(client.snatClient) } return nil } -func DeleteSnatEndpoint(client *OVSEndpointClient) error { - if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns { - return client.snatClient.DeleteSnatEndpoint() +func OVSDeleteSnatEndpoint(client *OVSEndpointClient) error { + if ovsEnableSnat(client) { + return DeleteSnatEndpoint(client.snatClient) } return nil } -func DeleteSnatEndpointRules(client *OVSEndpointClient) { - if client.allowInboundFromHostToNC { - client.snatClient.DeleteInboundFromHostToNC() - } - - if client.allowInboundFromNCToHost { - client.snatClient.DeleteInboundFromNCToHost() - } +func OVSDeleteSnatEndpointRules(client *OVSEndpointClient) { + DeleteSnatEndpointRules(client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost) } From 2d854b8df180981c05ca894031b130950b4928e1 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 16:54:59 -0700 Subject: [PATCH 60/86] Check to see if snat triggered --- network/native_endpoint_snatroute_linux.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/network/native_endpoint_snatroute_linux.go b/network/native_endpoint_snatroute_linux.go index 3c66c2f483..a74fb08eb5 100644 --- a/network/native_endpoint_snatroute_linux.go +++ b/network/native_endpoint_snatroute_linux.go @@ -1,6 +1,7 @@ package network import ( + "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/snat" ) @@ -8,6 +9,7 @@ func nativeEnableSnat(client *NativeEndpointClient) bool { return client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns } func NativeNewSnatClient(client *NativeEndpointClient, snatBridgeIP string, localIP string, epInfo *EndpointInfo) { + log.Printf("[native snat] %t %t %t %t", client.enableSnatOnHost, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.enableSnatForDns) if nativeEnableSnat(client) { client.snatClient = snat.NewSnatClient( GetSnatHostIfName(epInfo), From 8f2db2ff7b01161f8b3042d51d08a653997806cc Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 22 Jul 2022 10:09:30 -0700 Subject: [PATCH 61/86] Snat behaviors specific to ovs/native --- network/native_endpoint_snatroute_linux.go | 32 +++++++++++----------- network/native_endpointclient_linux.go | 16 +++++------ network/ovs_endpoint_snatroute_linux.go | 32 +++++++++++----------- network/ovs_endpointclient_linux.go | 16 +++++------ 4 files changed, 48 insertions(+), 48 deletions(-) diff --git a/network/native_endpoint_snatroute_linux.go b/network/native_endpoint_snatroute_linux.go index a74fb08eb5..3da45fcd8c 100644 --- a/network/native_endpoint_snatroute_linux.go +++ b/network/native_endpoint_snatroute_linux.go @@ -5,12 +5,12 @@ import ( "github.com/Azure/azure-container-networking/network/snat" ) -func nativeEnableSnat(client *NativeEndpointClient) bool { +func (client *NativeEndpointClient) isSnatEnabled() bool { return client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns } -func NativeNewSnatClient(client *NativeEndpointClient, snatBridgeIP string, localIP string, epInfo *EndpointInfo) { +func (client *NativeEndpointClient) NewSnatClient(snatBridgeIP string, localIP string, epInfo *EndpointInfo) { log.Printf("[native snat] %t %t %t %t", client.enableSnatOnHost, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.enableSnatForDns) - if nativeEnableSnat(client) { + if client.isSnatEnabled() { client.snatClient = snat.NewSnatClient( GetSnatHostIfName(epInfo), GetSnatContIfName(epInfo), @@ -23,8 +23,8 @@ func NativeNewSnatClient(client *NativeEndpointClient, snatBridgeIP string, loca ) } } -func NativeAddSnatEndpoint(client *NativeEndpointClient) error { - if nativeEnableSnat(client) { +func (client *NativeEndpointClient) AddSnatEndpoint() error { + if client.isSnatEnabled() { if err := AddSnatEndpoint(client.snatClient); err != nil { return err } @@ -32,8 +32,8 @@ func NativeAddSnatEndpoint(client *NativeEndpointClient) error { return nil } -func NativeAddSnatEndpointRules(client *NativeEndpointClient) error { - if nativeEnableSnat(client) { +func (client *NativeEndpointClient) AddSnatEndpointRules() error { + if client.isSnatEnabled() { // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge if err := AddSnatEndpointRules(client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.netlink, client.plClient); err != nil { return err @@ -43,38 +43,38 @@ func NativeAddSnatEndpointRules(client *NativeEndpointClient) error { return nil } -func NativeMoveSnatEndpointToContainerNS(client *NativeEndpointClient, netnsPath string, nsID uintptr) error { - if nativeEnableSnat(client) { +func (client *NativeEndpointClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { + if client.isSnatEnabled() { return MoveSnatEndpointToContainerNS(client.snatClient, netnsPath, nsID) } return nil } -func NativeSetupSnatContainerInterface(client *NativeEndpointClient) error { - if nativeEnableSnat(client) { +func (client *NativeEndpointClient) SetupSnatContainerInterface() error { + if client.isSnatEnabled() { return SetupSnatContainerInterface(client.snatClient) } return nil } -func NativeConfigureSnatContainerInterface(client *NativeEndpointClient) error { - if nativeEnableSnat(client) { +func (client *NativeEndpointClient) ConfigureSnatContainerInterface() error { + if client.isSnatEnabled() { return ConfigureSnatContainerInterface(client.snatClient) } return nil } -func NativeDeleteSnatEndpoint(client *NativeEndpointClient) error { - if nativeEnableSnat(client) { +func (client *NativeEndpointClient) DeleteSnatEndpoint() error { + if client.isSnatEnabled() { return DeleteSnatEndpoint(client.snatClient) } return nil } -func NativeDeleteSnatEndpointRules(client *NativeEndpointClient) { +func (client *NativeEndpointClient) DeleteSnatEndpointRules() { DeleteSnatEndpointRules(client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost) } diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 8ff77b24e8..e664e54594 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -83,7 +83,7 @@ func NewNativeEndpointClient( netUtilsClient: networkutils.NewNetworkUtils(nl, plc), } - NativeNewSnatClient(client, nw.SnatBridgeIP, localIP, ep) + client.NewSnatClient(nw.SnatBridgeIP, localIP, ep) return client @@ -96,7 +96,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if err != nil { return err } - if err := NativeAddSnatEndpoint(client); err != nil { + if err := client.AddSnatEndpoint(); err != nil { return errors.Wrap(err, "failed to add snat endpoint") } // VNET Namespace @@ -211,21 +211,21 @@ func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { } func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { - if err := NativeAddSnatEndpointRules(client); err != nil { + if err := client.AddSnatEndpointRules(); err != nil { return errors.Wrap(err, "failed to add snat endpoint rules") } return nil } func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { - NativeDeleteSnatEndpointRules(client) + client.DeleteSnatEndpointRules() } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { return errors.Wrap(err, "failed to move endpoint to container ns") } - if err := NativeMoveSnatEndpointToContainerNS(client, epInfo.NetNsPath, nsID); err != nil { + if err := client.MoveSnatEndpointToContainerNS(epInfo.NetNsPath, nsID); err != nil { return errors.Wrap(err, "failed to move snat endpoint to container ns") } return nil @@ -237,7 +237,7 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf } client.containerVethName = epInfo.IfName - if err := NativeSetupSnatContainerInterface(client); err != nil { + if err := client.SetupSnatContainerInterface(); err != nil { return errors.Wrap(err, "failed to setup snat container interface") } return nil @@ -251,7 +251,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return err } - if err = NativeConfigureSnatContainerInterface(client); err != nil { + if err = client.ConfigureSnatContainerInterface(); err != nil { return errors.Wrap(err, "failed to configure snat container interface") } @@ -388,7 +388,7 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { - if err := NativeDeleteSnatEndpoint(client); err != nil { + if err := client.DeleteSnatEndpoint(); err != nil { return errors.Wrap(err, "failed to delete snat endpoint") } diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index a42f28b74c..d2c23278e5 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -4,11 +4,11 @@ import ( "github.com/Azure/azure-container-networking/network/snat" ) -func ovsEnableSnat(client *OVSEndpointClient) bool { +func (client *OVSEndpointClient) isSnatEnabled() bool { return client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns } -func OVSNewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP string, epInfo *EndpointInfo) { - if ovsEnableSnat(client) { +func (client *OVSEndpointClient) NewSnatClient(snatBridgeIP string, localIP string, epInfo *EndpointInfo) { + if client.isSnatEnabled() { client.snatClient = snat.NewSnatClient( GetSnatHostIfName(epInfo), GetSnatContIfName(epInfo), @@ -21,8 +21,8 @@ func OVSNewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP st ) } } -func OVSAddSnatEndpoint(client *OVSEndpointClient) error { - if ovsEnableSnat(client) { +func (client *OVSEndpointClient) AddSnatEndpoint() error { + if client.isSnatEnabled() { if err := AddSnatEndpoint(client.snatClient); err != nil { return err } @@ -33,8 +33,8 @@ func OVSAddSnatEndpoint(client *OVSEndpointClient) error { return nil } -func OVSAddSnatEndpointRules(client *OVSEndpointClient) error { - if ovsEnableSnat(client) { +func (client *OVSEndpointClient) AddSnatEndpointRules() error { + if client.isSnatEnabled() { // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge if err := AddSnatEndpointRules(client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.netlink, client.plClient); err != nil { return err @@ -47,38 +47,38 @@ func OVSAddSnatEndpointRules(client *OVSEndpointClient) error { return nil } -func OVSMoveSnatEndpointToContainerNS(client *OVSEndpointClient, netnsPath string, nsID uintptr) error { - if ovsEnableSnat(client) { +func (client *OVSEndpointClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { + if client.isSnatEnabled() { return MoveSnatEndpointToContainerNS(client.snatClient, netnsPath, nsID) } return nil } -func OVSSetupSnatContainerInterface(client *OVSEndpointClient) error { - if ovsEnableSnat(client) { +func (client *OVSEndpointClient) SetupSnatContainerInterface() error { + if client.isSnatEnabled() { return SetupSnatContainerInterface(client.snatClient) } return nil } -func OVSConfigureSnatContainerInterface(client *OVSEndpointClient) error { - if ovsEnableSnat(client) { +func (client *OVSEndpointClient) ConfigureSnatContainerInterface() error { + if client.isSnatEnabled() { return ConfigureSnatContainerInterface(client.snatClient) } return nil } -func OVSDeleteSnatEndpoint(client *OVSEndpointClient) error { - if ovsEnableSnat(client) { +func (client *OVSEndpointClient) DeleteSnatEndpoint() error { + if client.isSnatEnabled() { return DeleteSnatEndpoint(client.snatClient) } return nil } -func OVSDeleteSnatEndpointRules(client *OVSEndpointClient) { +func (client *OVSEndpointClient) DeleteSnatEndpointRules() { DeleteSnatEndpointRules(client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost) } diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 84dc19e27e..e89203e363 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -72,7 +72,7 @@ func NewOVSEndpointClient( } NewInfraVnetClient(client, epInfo.Id[:7]) - NewSnatClient(client, nw.SnatBridgeIP, localIP, epInfo) + client.NewSnatClient(nw.SnatBridgeIP, localIP, epInfo) return client } @@ -91,7 +91,7 @@ func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { client.containerMac = containerIf.HardwareAddr.String() - if err := AddSnatEndpoint(client); err != nil { + if err := client.AddSnatEndpoint(); err != nil { return err } @@ -149,7 +149,7 @@ func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { return err } - return AddSnatEndpointRules(client) + return client.AddSnatEndpointRules() } func (client *OVSEndpointClient) DeleteEndpointRules(ep *endpoint) { @@ -183,7 +183,7 @@ func (client *OVSEndpointClient) DeleteEndpointRules(ep *endpoint) { log.Printf("[ovs] Deletion of interface %v from bridge %v failed", client.hostVethName, client.bridgeName) } - DeleteSnatEndpointRules(client) + client.DeleteSnatEndpointRules() DeleteInfraVnetEndpointRules(client, ep, hostPort) } @@ -194,7 +194,7 @@ func (client *OVSEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo return err } - if err := MoveSnatEndpointToContainerNS(client, epInfo.NetNsPath, nsID); err != nil { + if err := client.MoveSnatEndpointToContainerNS(epInfo.NetNsPath, nsID); err != nil { return err } @@ -209,7 +209,7 @@ func (client *OVSEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) client.containerVethName = epInfo.IfName - if err := SetupSnatContainerInterface(client); err != nil { + if err := client.SetupSnatContainerInterface(); err != nil { return err } @@ -222,7 +222,7 @@ func (client *OVSEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *E return err } - if err := ConfigureSnatContainerInterface(client); err != nil { + if err := client.ConfigureSnatContainerInterface(); err != nil { return err } @@ -241,6 +241,6 @@ func (client *OVSEndpointClient) DeleteEndpoints(ep *endpoint) error { return err } - DeleteSnatEndpoint(client) + client.DeleteSnatEndpoint() return DeleteInfraVnetEndpoint(client, ep.Id[:7]) } From faef1abbdec8a078cef79acd370b28abd1b7a702 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 22 Jul 2022 10:58:29 -0700 Subject: [PATCH 62/86] Pass the pointer Add/Delete ok --- network/endpoint_snatroute_linux.go | 14 +++++++------- network/native_endpoint_snatroute_linux.go | 14 +++++++------- network/ovs_endpoint_snatroute_linux.go | 14 +++++++------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/network/endpoint_snatroute_linux.go b/network/endpoint_snatroute_linux.go index 7958b833ba..d57f625533 100644 --- a/network/endpoint_snatroute_linux.go +++ b/network/endpoint_snatroute_linux.go @@ -16,11 +16,11 @@ func GetSnatHostIfName(epInfo *EndpointInfo) string { func GetSnatContIfName(epInfo *EndpointInfo) string { return fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7]) } -func AddSnatEndpoint(snatClient snat.SnatClient) error { +func AddSnatEndpoint(snatClient *snat.SnatClient) error { return snatClient.CreateSnatEndpoint() } -func AddSnatEndpointRules(snatClient snat.SnatClient, hostToNC, NCToHost bool, nl netlink.NetlinkInterface, plc platform.ExecClient) error { +func AddSnatEndpointRules(snatClient *snat.SnatClient, hostToNC, NCToHost bool, nl netlink.NetlinkInterface, plc platform.ExecClient) error { // Allow specific Private IPs via Snat Bridge if err := snatClient.AllowIPAddressesOnSnatBridge(); err != nil { return err @@ -47,23 +47,23 @@ func AddSnatEndpointRules(snatClient snat.SnatClient, hostToNC, NCToHost bool, n return nil } -func MoveSnatEndpointToContainerNS(snatClient snat.SnatClient, netnsPath string, nsID uintptr) error { +func MoveSnatEndpointToContainerNS(snatClient *snat.SnatClient, netnsPath string, nsID uintptr) error { return snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID) } -func SetupSnatContainerInterface(snatClient snat.SnatClient) error { +func SetupSnatContainerInterface(snatClient *snat.SnatClient) error { return snatClient.SetupSnatContainerInterface() } -func ConfigureSnatContainerInterface(snatClient snat.SnatClient) error { +func ConfigureSnatContainerInterface(snatClient *snat.SnatClient) error { return snatClient.ConfigureSnatContainerInterface() } -func DeleteSnatEndpoint(snatClient snat.SnatClient) error { +func DeleteSnatEndpoint(snatClient *snat.SnatClient) error { return snatClient.DeleteSnatEndpoint() } -func DeleteSnatEndpointRules(snatClient snat.SnatClient, hostToNC, NCToHost bool) { +func DeleteSnatEndpointRules(snatClient *snat.SnatClient, hostToNC, NCToHost bool) { if hostToNC { snatClient.DeleteInboundFromHostToNC() } diff --git a/network/native_endpoint_snatroute_linux.go b/network/native_endpoint_snatroute_linux.go index 3da45fcd8c..d2eaa5bb16 100644 --- a/network/native_endpoint_snatroute_linux.go +++ b/network/native_endpoint_snatroute_linux.go @@ -25,7 +25,7 @@ func (client *NativeEndpointClient) NewSnatClient(snatBridgeIP string, localIP s } func (client *NativeEndpointClient) AddSnatEndpoint() error { if client.isSnatEnabled() { - if err := AddSnatEndpoint(client.snatClient); err != nil { + if err := AddSnatEndpoint(&client.snatClient); err != nil { return err } } @@ -35,7 +35,7 @@ func (client *NativeEndpointClient) AddSnatEndpoint() error { func (client *NativeEndpointClient) AddSnatEndpointRules() error { if client.isSnatEnabled() { // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge - if err := AddSnatEndpointRules(client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.netlink, client.plClient); err != nil { + if err := AddSnatEndpointRules(&client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.netlink, client.plClient); err != nil { return err } } @@ -45,7 +45,7 @@ func (client *NativeEndpointClient) AddSnatEndpointRules() error { func (client *NativeEndpointClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { if client.isSnatEnabled() { - return MoveSnatEndpointToContainerNS(client.snatClient, netnsPath, nsID) + return MoveSnatEndpointToContainerNS(&client.snatClient, netnsPath, nsID) } return nil @@ -53,7 +53,7 @@ func (client *NativeEndpointClient) MoveSnatEndpointToContainerNS(netnsPath stri func (client *NativeEndpointClient) SetupSnatContainerInterface() error { if client.isSnatEnabled() { - return SetupSnatContainerInterface(client.snatClient) + return SetupSnatContainerInterface(&client.snatClient) } return nil @@ -61,7 +61,7 @@ func (client *NativeEndpointClient) SetupSnatContainerInterface() error { func (client *NativeEndpointClient) ConfigureSnatContainerInterface() error { if client.isSnatEnabled() { - return ConfigureSnatContainerInterface(client.snatClient) + return ConfigureSnatContainerInterface(&client.snatClient) } return nil @@ -69,12 +69,12 @@ func (client *NativeEndpointClient) ConfigureSnatContainerInterface() error { func (client *NativeEndpointClient) DeleteSnatEndpoint() error { if client.isSnatEnabled() { - return DeleteSnatEndpoint(client.snatClient) + return DeleteSnatEndpoint(&client.snatClient) } return nil } func (client *NativeEndpointClient) DeleteSnatEndpointRules() { - DeleteSnatEndpointRules(client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost) + DeleteSnatEndpointRules(&client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost) } diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index d2c23278e5..09dce057d2 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -23,7 +23,7 @@ func (client *OVSEndpointClient) NewSnatClient(snatBridgeIP string, localIP stri } func (client *OVSEndpointClient) AddSnatEndpoint() error { if client.isSnatEnabled() { - if err := AddSnatEndpoint(client.snatClient); err != nil { + if err := AddSnatEndpoint(&client.snatClient); err != nil { return err } if err := client.ovsctlClient.AddPortOnOVSBridge(snat.AzureSnatVeth1, client.bridgeName, 0); err != nil { @@ -36,7 +36,7 @@ func (client *OVSEndpointClient) AddSnatEndpoint() error { func (client *OVSEndpointClient) AddSnatEndpointRules() error { if client.isSnatEnabled() { // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge - if err := AddSnatEndpointRules(client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.netlink, client.plClient); err != nil { + if err := AddSnatEndpointRules(&client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.netlink, client.plClient); err != nil { return err } if err := AddStaticRoute(client.netlink, client.netioshim, snat.ImdsIP, client.bridgeName); err != nil { @@ -49,7 +49,7 @@ func (client *OVSEndpointClient) AddSnatEndpointRules() error { func (client *OVSEndpointClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { if client.isSnatEnabled() { - return MoveSnatEndpointToContainerNS(client.snatClient, netnsPath, nsID) + return MoveSnatEndpointToContainerNS(&client.snatClient, netnsPath, nsID) } return nil @@ -57,7 +57,7 @@ func (client *OVSEndpointClient) MoveSnatEndpointToContainerNS(netnsPath string, func (client *OVSEndpointClient) SetupSnatContainerInterface() error { if client.isSnatEnabled() { - return SetupSnatContainerInterface(client.snatClient) + return SetupSnatContainerInterface(&client.snatClient) } return nil @@ -65,7 +65,7 @@ func (client *OVSEndpointClient) SetupSnatContainerInterface() error { func (client *OVSEndpointClient) ConfigureSnatContainerInterface() error { if client.isSnatEnabled() { - return ConfigureSnatContainerInterface(client.snatClient) + return ConfigureSnatContainerInterface(&client.snatClient) } return nil @@ -73,12 +73,12 @@ func (client *OVSEndpointClient) ConfigureSnatContainerInterface() error { func (client *OVSEndpointClient) DeleteSnatEndpoint() error { if client.isSnatEnabled() { - return DeleteSnatEndpoint(client.snatClient) + return DeleteSnatEndpoint(&client.snatClient) } return nil } func (client *OVSEndpointClient) DeleteSnatEndpointRules() { - DeleteSnatEndpointRules(client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost) + DeleteSnatEndpointRules(&client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost) } From 9fa80a9339c96a29089e92d5317dbdd509a254e1 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 22 Jul 2022 14:16:34 -0700 Subject: [PATCH 63/86] Renaming to make consts public --- network/snat/snat_linux.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go index 48cdd8d7e6..760eb08dca 100644 --- a/network/snat/snat_linux.go +++ b/network/snat/snat_linux.go @@ -18,7 +18,7 @@ import ( ) const ( - azureSnatVeth0 = "azSnatveth0" + AzureSnatVeth0 = "azSnatveth0" AzureSnatVeth1 = "azSnatveth1" azureSnatIfName = "eth1" SnatBridgeName = "azSnatbr" @@ -133,7 +133,7 @@ func (client *SnatClient) BlockIPAddressesOnSnatBridge() error { Move container veth inside container network namespace **/ func (client *SnatClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { - log.Printf("[ovs] Setting link %v netns %v.", client.containerSnatVethName, netnsPath) + log.Printf("[snat] Setting link %v netns %v.", client.containerSnatVethName, netnsPath) err := client.netlink.SetLinkNetNs(client.containerSnatVethName, nsID) if err != nil { return newErrorSnatClient(err.Error()) @@ -167,7 +167,7 @@ func getNCLocalAndGatewayIP(client *SnatClient) (net.IP, net.IP) { func (client *SnatClient) AllowInboundFromHostToNC() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) - // Create CNI Ouptut chain + // Create CNI Output chain if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil { log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err) return newErrorSnatClient(err.Error()) @@ -394,13 +394,13 @@ func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac s } log.Printf("Drop ARP for snat bridge ip: %s", snatBridgeIP) - if err := client.dropArpForSnatBridgeApipaRange(snatBridgeIP, azureSnatVeth0); err != nil { + if err := client.dropArpForSnatBridgeApipaRange(snatBridgeIP, AzureSnatVeth0); err != nil { return err } // Create a veth pair. One end of veth will be attached to ovs bridge and other end // of veth will be attached to linux bridge - _, err = net.InterfaceByName(azureSnatVeth0) + _, err = net.InterfaceByName(AzureSnatVeth0) if err == nil { log.Printf("Azure snat veth already exists") return nil @@ -415,7 +415,7 @@ func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac s vethLink := netlink.VEthLink{ LinkInfo: netlink.LinkInfo{ Type: netlink.LINK_TYPE_VETH, - Name: azureSnatVeth0, + Name: AzureSnatVeth0, }, PeerName: AzureSnatVeth1, } @@ -427,7 +427,7 @@ func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac s } //nolint - if err = nuc.DisableRAForInterface(azureSnatVeth0); err != nil { + if err = nuc.DisableRAForInterface(AzureSnatVeth0); err != nil { return err } @@ -449,11 +449,11 @@ func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac s return newErrorSnatClient(err.Error()) } - if err = client.netlink.SetLinkState(azureSnatVeth0, true); err != nil { + if err = client.netlink.SetLinkState(AzureSnatVeth0, true); err != nil { return newErrorSnatClient(err.Error()) } - if err = client.netlink.SetLinkMaster(azureSnatVeth0, SnatBridgeName); err != nil { + if err = client.netlink.SetLinkMaster(AzureSnatVeth0, SnatBridgeName); err != nil { return newErrorSnatClient(err.Error()) } From 0bc785af8b5a012d14d8809e18e8c1d293925f51 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 22 Jul 2022 15:38:11 -0700 Subject: [PATCH 64/86] Breaking, moving ovs specific parts of snat to ovs --- network/ovs_endpoint_snatroute_linux.go | 67 ++++++++++++++++++++++++- network/snat/snat_linux.go | 66 ++++-------------------- network/snat/snat_linux_test.go | 4 +- 3 files changed, 77 insertions(+), 60 deletions(-) diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index 09dce057d2..c505d3659c 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -1,9 +1,20 @@ package network import ( + "net" + + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/snat" ) +// Communication between ovs switch and snat bridge, master of azuresnatveth0 is snat bridge itself +const ( + azureSnatVeth0 = "azSnatveth0" + azureSnatVeth1 = "azSnatveth1" +) + func (client *OVSEndpointClient) isSnatEnabled() bool { return client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns } @@ -26,7 +37,61 @@ func (client *OVSEndpointClient) AddSnatEndpoint() error { if err := AddSnatEndpoint(&client.snatClient); err != nil { return err } - if err := client.ovsctlClient.AddPortOnOVSBridge(snat.AzureSnatVeth1, client.bridgeName, 0); err != nil { + if err := client.ovsctlClient.AddPortOnOVSBridge(azureSnatVeth1, client.bridgeName, 0); err != nil { + return err + } + snatClient := client.snatClient + + // Separated + log.Printf("Drop ARP for snat bridge ip: %s", snatClient.SnatBridgeIP) + if err := client.snatClient.DropArpForSnatBridgeApipaRange(snatClient.SnatBridgeIP, azureSnatVeth0); err != nil { + return err + } + + // Create a veth pair. One end of veth will be attached to ovs bridge and other end + // of veth will be attached to linux bridge + _, err := net.InterfaceByName(azureSnatVeth0) + if err == nil { + log.Printf("Azure snat veth already exists") + return nil + } + + // Separated + vethLink := netlink.VEthLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_VETH, + Name: azureSnatVeth0, + }, + PeerName: azureSnatVeth1, + } + + err = client.netlink.AddLink(&vethLink) + if err != nil { + log.Printf("[net] Failed to create veth pair, err:%v.", err) + return err + } + // Added in + nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) + //nolint + if err = nuc.DisableRAForInterface(azureSnatVeth0); err != nil { + return err + } + + //nolint + if err = nuc.DisableRAForInterface(azureSnatVeth1); err != nil { + return err + } + + // Separated + if err = client.netlink.SetLinkState(azureSnatVeth0, true); err != nil { + return err + } + + if err = client.netlink.SetLinkMaster(azureSnatVeth0, snat.SnatBridgeName); err != nil { + return err + } + + if err = client.netlink.SetLinkState(azureSnatVeth1, true); err != nil { return err } } diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go index 760eb08dca..76bd7b6151 100644 --- a/network/snat/snat_linux.go +++ b/network/snat/snat_linux.go @@ -18,8 +18,6 @@ import ( ) const ( - AzureSnatVeth0 = "azSnatveth0" - AzureSnatVeth1 = "azSnatveth1" azureSnatIfName = "eth1" SnatBridgeName = "azSnatbr" ImdsIP = "169.254.169.254/32" @@ -40,7 +38,7 @@ type SnatClient struct { hostPrimaryMac string containerSnatVethName string localIP string - snatBridgeIP string + SnatBridgeIP string SkipAddressesFromBlock []string netlink netlink.NetlinkInterface @@ -62,7 +60,7 @@ func NewSnatClient(hostIfName string, hostSnatVethName: hostIfName, containerSnatVethName: contIfName, localIP: localIP, - snatBridgeIP: snatBridgeIP, + SnatBridgeIP: snatBridgeIP, hostPrimaryMac: hostPrimaryMac, netlink: nl, @@ -78,13 +76,13 @@ func NewSnatClient(hostIfName string, func (client *SnatClient) CreateSnatEndpoint() error { // Create linux Bridge for outbound connectivity - if err := client.createSnatBridge(client.snatBridgeIP, client.hostPrimaryMac); err != nil { + if err := client.createSnatBridge(client.SnatBridgeIP, client.hostPrimaryMac); err != nil { log.Printf("creating snat bridge failed with error %v", err) return err } // SNAT Rule to masquerade packets destined to non-vnet ip - if err := client.addMasqueradeRule(client.snatBridgeIP); err != nil { + if err := client.addMasqueradeRule(client.SnatBridgeIP); err != nil { log.Printf("Adding snat rule failed with error %v", err) return err } @@ -156,7 +154,7 @@ func (client *SnatClient) SetupSnatContainerInterface() error { } func getNCLocalAndGatewayIP(client *SnatClient) (net.IP, net.IP) { - bridgeIP, _, _ := net.ParseCIDR(client.snatBridgeIP) + bridgeIP, _, _ := net.ParseCIDR(client.SnatBridgeIP) containerIP, _, _ := net.ParseCIDR(client.localIP) return bridgeIP, containerIP } @@ -356,7 +354,7 @@ func (client *SnatClient) setBridgeMac(hostPrimaryMac string) error { return err } -func (client *SnatClient) dropArpForSnatBridgeApipaRange(snatBridgeIP, azSnatVethIfName string) error { +func (client *SnatClient) DropArpForSnatBridgeApipaRange(snatBridgeIP, azSnatVethIfName string) error { var err error _, ipCidr, _ := net.ParseCIDR(snatBridgeIP) if err = ebtables.SetArpDropRuleForIpCidr(ipCidr.String(), azSnatVethIfName); err != nil { @@ -392,49 +390,14 @@ func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac s if err := client.setBridgeMac(hostPrimaryMac); err != nil { return err } - - log.Printf("Drop ARP for snat bridge ip: %s", snatBridgeIP) - if err := client.dropArpForSnatBridgeApipaRange(snatBridgeIP, AzureSnatVeth0); err != nil { - return err - } - - // Create a veth pair. One end of veth will be attached to ovs bridge and other end - // of veth will be attached to linux bridge - _, err = net.InterfaceByName(AzureSnatVeth0) - if err == nil { - log.Printf("Azure snat veth already exists") - return nil - } + // Drop ARP and InterfaceByName were here nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) //nolint if err = nuc.DisableRAForInterface(SnatBridgeName); err != nil { return err } - - vethLink := netlink.VEthLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_VETH, - Name: AzureSnatVeth0, - }, - PeerName: AzureSnatVeth1, - } - - err = client.netlink.AddLink(&vethLink) - if err != nil { - log.Printf("[net] Failed to create veth pair, err:%v.", err) - return err - } - - //nolint - if err = nuc.DisableRAForInterface(AzureSnatVeth0); err != nil { - return err - } - - //nolint - if err = nuc.DisableRAForInterface(AzureSnatVeth1); err != nil { - return err - } + // VethLink, AddLink, and Disable RA for azureSnatVeth0/1 were here log.Printf("Assigning %v on snat bridge", snatBridgeIP) @@ -448,18 +411,7 @@ func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac s if err = client.netlink.SetLinkState(SnatBridgeName, true); err != nil { return newErrorSnatClient(err.Error()) } - - if err = client.netlink.SetLinkState(AzureSnatVeth0, true); err != nil { - return newErrorSnatClient(err.Error()) - } - - if err = client.netlink.SetLinkMaster(AzureSnatVeth0, SnatBridgeName); err != nil { - return newErrorSnatClient(err.Error()) - } - - if err = client.netlink.SetLinkState(AzureSnatVeth1, true); err != nil { - return newErrorSnatClient(err.Error()) - } + // Set link state of azure snatveth0/1 and link master of azuresnatveth0 set to snatbridgename return nil } diff --git a/network/snat/snat_linux_test.go b/network/snat/snat_linux_test.go index 69fe80bb5e..e0209c1201 100644 --- a/network/snat/snat_linux_test.go +++ b/network/snat/snat_linux_test.go @@ -20,7 +20,7 @@ func TestMain(m *testing.M) { func TestAllowInboundFromHostToNC(t *testing.T) { nl := netlink.NewNetlink() client := &SnatClient{ - snatBridgeIP: "169.254.0.1/16", + SnatBridgeIP: "169.254.0.1/16", localIP: "169.254.0.4/16", containerSnatVethName: anyInterface, netlink: nl, @@ -67,7 +67,7 @@ func TestAllowInboundFromHostToNC(t *testing.T) { func TestAllowInboundFromNCToHost(t *testing.T) { nl := netlink.NewNetlink() client := &SnatClient{ - snatBridgeIP: "169.254.0.1/16", + SnatBridgeIP: "169.254.0.1/16", localIP: "169.254.0.4/16", containerSnatVethName: anyInterface, netlink: nl, From 71dc1e4733e6f86746a6f1329d19621e94204267 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 22 Jul 2022 15:38:28 -0700 Subject: [PATCH 65/86] Remove enable infra vnet (Tests ok) Tested: Allow Host to NC only Allow NC to Host only Allow both Wget Ping between containers Warning: Enable snat is still hard coded to true!!! --- network/native_endpointclient_linux.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index e664e54594..d54bd579fa 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -62,6 +62,8 @@ func NewNativeEndpointClient( vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, vlanid) vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) + + log.Printf("[native snat constructor] %t %t %t %t", ep.EnableSnatOnHost, ep.AllowInboundFromHostToNC, ep.AllowInboundFromNCToHost, ep.EnableSnatForDns) client := &NativeEndpointClient{ bridgeName: nw.extIf.BridgeName, eth0VethName: nw.extIf.Name, @@ -72,7 +74,6 @@ func NewNativeEndpointClient( vnetNSName: vnetNSName, vlanID: vlanid, enableSnatOnHost: ep.EnableSnatOnHost, - enableInfraVnet: ep.EnableInfraVnet, allowInboundFromHostToNC: ep.AllowInboundFromHostToNC, allowInboundFromNCToHost: ep.AllowInboundFromNCToHost, enableSnatForDns: ep.EnableSnatForDns, From 49eb30c005337f6785677e344e9db195d1339293 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 25 Jul 2022 14:34:00 -0700 Subject: [PATCH 66/86] Move add port to after exists() check --- network/ovs_endpoint_snatroute_linux.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index c505d3659c..d608fa72b4 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -37,9 +37,7 @@ func (client *OVSEndpointClient) AddSnatEndpoint() error { if err := AddSnatEndpoint(&client.snatClient); err != nil { return err } - if err := client.ovsctlClient.AddPortOnOVSBridge(azureSnatVeth1, client.bridgeName, 0); err != nil { - return err - } + snatClient := client.snatClient // Separated @@ -94,6 +92,10 @@ func (client *OVSEndpointClient) AddSnatEndpoint() error { if err = client.netlink.SetLinkState(azureSnatVeth1, true); err != nil { return err } + + if err := client.ovsctlClient.AddPortOnOVSBridge(azureSnatVeth1, client.bridgeName, 0); err != nil { + return err + } } return nil } From ba901c592ad7d04e8637ef64ada1f2a890e2a501 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 26 Jul 2022 09:43:51 -0700 Subject: [PATCH 67/86] Moved netns interface to caller, generalized tests Tests ok, Native ok --- netns/mocknetns.go | 71 ------ network/native_endpointclient_linux.go | 9 +- network/native_endpointclient_linux_test.go | 231 ++++++++++++++------ network/netnsinterface.go | 9 - 4 files changed, 175 insertions(+), 145 deletions(-) delete mode 100644 netns/mocknetns.go delete mode 100644 network/netnsinterface.go diff --git a/netns/mocknetns.go b/netns/mocknetns.go deleted file mode 100644 index 432fe03258..0000000000 --- a/netns/mocknetns.go +++ /dev/null @@ -1,71 +0,0 @@ -//go:build linux -// +build linux - -package netns - -import ( - "github.com/pkg/errors" -) - -const ( - Get int = 1 - GetFromName int = 2 - Set int = 3 - NewNamed int = 4 - DeleteNamed int = 5 -) - -var ErrorMock = errors.New("mock netns error") - -func newErrorMock(errStr string) error { - return errors.Wrap(ErrorMock, errStr) -} - -type MockNetns struct { - failMethod int - failMethod2 int - failMessage string -} - -func NewMock(failMethod, failMethod2 int, failMessage string) *MockNetns { - return &MockNetns{ - failMethod: failMethod, - failMethod2: failMethod2, - failMessage: failMessage, - } -} - -func (f *MockNetns) Get() (int, error) { - if f.failMethod == Get || f.failMethod2 == Get { - return 0, newErrorMock(f.failMessage) - } - return 1, nil -} - -func (f *MockNetns) GetFromName(name string) (int, error) { - if f.failMethod == GetFromName || f.failMethod2 == GetFromName { - return 0, newErrorMock(f.failMessage) - } - return 1, nil -} - -func (f *MockNetns) Set(handle int) error { - if f.failMethod == Set || f.failMethod2 == Set { - return newErrorMock(f.failMessage) - } - return nil -} - -func (f *MockNetns) NewNamed(name string) (int, error) { - if f.failMethod == NewNamed || f.failMethod2 == NewNamed { - return 0, newErrorMock(f.failMessage) - } - return 1, nil -} - -func (f *MockNetns) DeleteNamed(name string) error { - if f.failMethod == DeleteNamed || f.failMethod2 == DeleteNamed { - return newErrorMock(f.failMessage) - } - return nil -} diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index d2cec73b7e..bc674ca8fd 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -19,6 +19,13 @@ const ( numDefaultRoutes = 2 // VNET NS, when no containers use it, has this many routes ) +type netnsClient interface { + Get() (fileDescriptor int, err error) + GetFromName(name string) (fileDescriptor int, err error) + Set(fileDescriptor int) (err error) + NewNamed(name string) (fileDescriptor int, err error) + DeleteNamed(name string) (err error) +} type NativeEndpointClient struct { eth0VethName string // So like eth0 vlanVethName string // So like eth0.1 @@ -33,7 +40,7 @@ type NativeEndpointClient struct { nw *network vlanID int - netnsClient NetnsInterface + netnsClient netnsClient netlink netlink.NetlinkInterface netioshim netio.NetIOInterface plClient platform.ExecClient diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 7ce99cb888..615737e16b 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -9,12 +9,66 @@ import ( "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" + "github.com/pkg/errors" "github.com/stretchr/testify/require" ) +var ErrorMock = errors.New("mock netns error") + +func newErrorMock(errStr string) error { + return errors.Wrap(ErrorMock, errStr) +} + +type mockNetns struct { + get func() (fileDescriptor int, err error) + getFromName func(name string) (fileDescriptor int, err error) + set func(fileDescriptor int) (err error) + newNamed func(name string) (fileDescriptor int, err error) + deleteNamed func(name string) (err error) +} + +func (netns *mockNetns) Get() (fileDescriptor int, err error) { + return netns.get() +} + +func (netns *mockNetns) GetFromName(name string) (fileDescriptor int, err error) { + return netns.getFromName(name) +} + +func (netns *mockNetns) Set(fileDescriptor int) (err error) { + return netns.set(fileDescriptor) +} + +func (netns *mockNetns) NewNamed(name string) (fileDescriptor int, err error) { + return netns.newNamed(name) +} + +func (netns *mockNetns) DeleteNamed(name string) (err error) { + return netns.deleteNamed(name) +} + +func defaultGet() (int, error) { + return 1, nil +} + +func defaultGetFromName(name string) (int, error) { + return 1, nil +} + +func defaultSet(handle int) error { + return nil +} + +func defaultNewNamed(name string) (int, error) { + return 1, nil +} + +func defaultDeleteNamed(name string) error { + return nil +} + func TestNativeAddEndpoints(t *testing.T) { nl := netlink.NewMockNetlink(false, "") plc := platform.NewMockExecClient(false) @@ -35,15 +89,23 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.GetFromName, netns.NewNamed, "netns failure"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: func(name string) (fileDescriptor int, err error) { + return 0, newErrorMock("netns failure") + }, + newNamed: func(name string) (fileDescriptor int, err error) { + return 0, newErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to create vnet ns: netns failure: " + netns.ErrorMock.Error(), + wantErrMsg: "failed to create vnet ns: netns failure: " + ErrorMock.Error(), }, { name: "Add endpoints with existing vnet ns", @@ -53,11 +115,17 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: defaultGetFromName, + newNamed: defaultNewNamed, + set: defaultSet, + deleteNamed: defaultDeleteNamed, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, wantErr: false, @@ -70,11 +138,17 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), - netlink: netlink.NewMockNetlink(true, "netlink fail"), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: defaultGetFromName, + newNamed: defaultNewNamed, + set: defaultSet, + deleteNamed: defaultDeleteNamed, + }, + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, wantErr: true, @@ -88,11 +162,19 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.GetFromName, 0, "netns fail"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 1), + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: func(name string) (fileDescriptor int, err error) { + return 0, newErrorMock("netns failure") + }, + newNamed: defaultNewNamed, + set: defaultSet, + deleteNamed: defaultDeleteNamed, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, wantErr: true, @@ -106,11 +188,17 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 1), + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: defaultGetFromName, + newNamed: defaultNewNamed, + set: defaultSet, + deleteNamed: defaultDeleteNamed, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, wantErr: true, @@ -124,15 +212,19 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.Get, 0, "netns failure"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + get: func() (fileDescriptor int, err error) { + return 0, newErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to get vm ns handle: netns failure: " + netns.ErrorMock.Error(), + wantErrMsg: "failed to get vm ns handle: netns failure: " + ErrorMock.Error(), }, { name: "Add endpoints NetNS Set fail", @@ -142,15 +234,25 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.Set, netns.GetFromName, "netns failure"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: func(name string) (fileDescriptor int, err error) { + return 0, newErrorMock("do not fail on this error") + }, + newNamed: defaultNewNamed, + set: func(fileDescriptor int) (err error) { + return newErrorMock("netns failure") + }, + deleteNamed: defaultDeleteNamed, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to set current ns to vm: netns failure: " + netns.ErrorMock.Error(), + wantErrMsg: "failed to set current ns to vm: netns failure: " + ErrorMock.Error(), }, } @@ -183,7 +285,6 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -200,7 +301,6 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -218,7 +318,6 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -273,11 +372,13 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + deleteNamed: defaultDeleteNamed, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ IPAddresses: IPAddresses, @@ -293,11 +394,15 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.DeleteNamed, 0, "netns failure"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + deleteNamed: func(name string) (err error) { + return newErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ IPAddresses: IPAddresses, @@ -313,18 +418,22 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.DeleteNamed, 0, "netns failure"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + deleteNamed: func(name string) (err error) { + return newErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ IPAddresses: IPAddresses, }, routesLeft: numDefaultRoutes + len(IPAddresses), wantErr: true, - wantErrMsg: "failed to delete namespace: netns failure: " + netns.ErrorMock.Error(), + wantErrMsg: "failed to delete namespace: netns failure: " + ErrorMock.Error(), }, } @@ -364,7 +473,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -389,7 +497,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -422,7 +529,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -448,7 +554,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -495,7 +600,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -521,7 +625,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), diff --git a/network/netnsinterface.go b/network/netnsinterface.go deleted file mode 100644 index 8f6311c386..0000000000 --- a/network/netnsinterface.go +++ /dev/null @@ -1,9 +0,0 @@ -package network - -type NetnsInterface interface { - Get() (fileDescriptor int, err error) - GetFromName(name string) (fileDescriptor int, err error) - Set(fileDescriptor int) (err error) - NewNamed(name string) (fileDescriptor int, err error) - DeleteNamed(name string) (err error) -} From 33f4941cccc9e7bad72382b398bf94489e7d5e90 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 27 Jul 2022 07:56:53 -0700 Subject: [PATCH 68/86] Typos --- network/native_endpointclient_linux.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index bc674ca8fd..5add1d5966 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -244,7 +244,7 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) if err = client.AddDefaultRoutes(client.vlanVethName); err != nil { - return errors.Wrap(err, "failed vnet ns add default/gateway routes (indempotent)") + return errors.Wrap(err, "failed vnet ns add default/gateway routes (idempotent)") } if err = client.AddDefaultArp(client.vlanVethName, azureMac); err != nil { return errors.Wrap(err, "failed vnet ns add default arp entry (idempotent)") @@ -362,7 +362,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint, routesLeft return nil } -// Helper function that allows executing a function with one parameter in a VM namespace +// Helper function that allows executing a function in a VM namespace // Does not work for process namespaces func ExecuteInNS(nsName string, f func() error) error { // Current namespace From 228ba638c05c1178d9f443770f0816b2f6f14021 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 27 Jul 2022 14:31:30 -0700 Subject: [PATCH 69/86] Reordered if statement, unwrapped arp Tests ok, ping ok, wget ok --- network/native_endpointclient_linux.go | 30 +++++++++++--------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 5add1d5966..2f1c637d88 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -112,21 +112,19 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { log.Printf("[native] Attempting to create %s link in VM NS", client.vlanVethName) // Create vlan veth deleteNSIfNotNilErr = vishnetlink.LinkAdd(link) - if deleteNSIfNotNilErr == nil { - // vlan veth was created successfully, so move the vlan veth you created - log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) - deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) - if deleteNSIfNotNilErr != nil { - if delErr := client.netlink.DeleteLink(client.vlanVethName); delErr != nil { - log.Errorf("deleting vlan veth failed on addendpoint failure") - } - return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") - } - } else { - // Any other error + if deleteNSIfNotNilErr != nil { + // Any failure to add the link should error (auto delete NS) return errors.Wrap(deleteNSIfNotNilErr, "failed to create vlan vnet link after making new ns") } - + // vlan veth was created successfully, so move the vlan veth you created + log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) + deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) + if deleteNSIfNotNilErr != nil { + if delErr := client.netlink.DeleteLink(client.vlanVethName); delErr != nil { + log.Errorf("deleting vlan veth failed on addendpoint failure") + } + return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") + } } else { log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanVethName) } @@ -321,11 +319,7 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) if err != nil { return errors.Wrap(err, "unable to parse mac") } - if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, - interfaceName, - virtualGwNet.IP, - hardwareAddr, - false); err != nil { + if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, interfaceName, virtualGwNet.IP, hardwareAddr, false); err != nil { return fmt.Errorf("adding arp entry failed: %w", err) } return nil From 5c2a77dac4ab3002441116344c8c4ff01a37503f Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 27 Jul 2022 15:14:03 -0700 Subject: [PATCH 70/86] Linted, wrapping errors --- network/endpoint_snatroute_linux.go | 53 +++++++++++++--------- network/native_endpoint_snatroute_linux.go | 8 ++-- network/native_endpointclient_linux.go | 13 ++---- network/ovs_endpoint_snatroute_linux.go | 21 +++++---- network/ovs_endpointclient_linux.go | 10 ++-- network/snat/snat_linux.go | 40 ++++++++-------- network/snat/snat_linux_test.go | 4 +- 7 files changed, 81 insertions(+), 68 deletions(-) diff --git a/network/endpoint_snatroute_linux.go b/network/endpoint_snatroute_linux.go index d57f625533..a10c562b21 100644 --- a/network/endpoint_snatroute_linux.go +++ b/network/endpoint_snatroute_linux.go @@ -3,72 +3,81 @@ package network import ( "fmt" + "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/snat" "github.com/Azure/azure-container-networking/platform" + "github.com/pkg/errors" ) func GetSnatHostIfName(epInfo *EndpointInfo) string { return fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7]) - } + func GetSnatContIfName(epInfo *EndpointInfo) string { return fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7]) } -func AddSnatEndpoint(snatClient *snat.SnatClient) error { - return snatClient.CreateSnatEndpoint() + +func AddSnatEndpoint(snatClient *snat.Client) error { + return errors.Wrap(snatClient.CreateSnatEndpoint(), "failed to add snat endpoint") } -func AddSnatEndpointRules(snatClient *snat.SnatClient, hostToNC, NCToHost bool, nl netlink.NetlinkInterface, plc platform.ExecClient) error { +func AddSnatEndpointRules(snatClient *snat.Client, hostToNC, ncToHost bool, nl netlink.NetlinkInterface, plc platform.ExecClient) error { // Allow specific Private IPs via Snat Bridge if err := snatClient.AllowIPAddressesOnSnatBridge(); err != nil { - return err + return errors.Wrap(err, "failed to allow ip addresses on snat bridge") } // Block Private IPs via Snat Bridge if err := snatClient.BlockIPAddressesOnSnatBridge(); err != nil { - return err + return errors.Wrap(err, "failed to block ip addresses on snat bridge") } nuc := networkutils.NewNetworkUtils(nl, plc) if err := nuc.EnableIPForwarding(snat.SnatBridgeName); err != nil { - return err + return errors.Wrap(err, "failed to enable ip forwarding") } if hostToNC { if err := snatClient.AllowInboundFromHostToNC(); err != nil { - return err + return errors.Wrap(err, "failed to allow inbound from host to nc") } } - if NCToHost { - return snatClient.AllowInboundFromNCToHost() + if ncToHost { + return errors.Wrap(snatClient.AllowInboundFromNCToHost(), "failed to allow inbound from nc to host") } return nil } -func MoveSnatEndpointToContainerNS(snatClient *snat.SnatClient, netnsPath string, nsID uintptr) error { - return snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID) +func MoveSnatEndpointToContainerNS(snatClient *snat.Client, netnsPath string, nsID uintptr) error { + return errors.Wrap(snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID), "failed to move snat endpoint to container ns") } -func SetupSnatContainerInterface(snatClient *snat.SnatClient) error { - return snatClient.SetupSnatContainerInterface() +func SetupSnatContainerInterface(snatClient *snat.Client) error { + return errors.Wrap(snatClient.SetupSnatContainerInterface(), "failed to setup snat container interface") } -func ConfigureSnatContainerInterface(snatClient *snat.SnatClient) error { - return snatClient.ConfigureSnatContainerInterface() +func ConfigureSnatContainerInterface(snatClient *snat.Client) error { + return errors.Wrap(snatClient.ConfigureSnatContainerInterface(), "failed to configure snat container interface") } -func DeleteSnatEndpoint(snatClient *snat.SnatClient) error { - return snatClient.DeleteSnatEndpoint() +func DeleteSnatEndpoint(snatClient *snat.Client) error { + return errors.Wrap(snatClient.DeleteSnatEndpoint(), "failed to delete snat endpoint") } -func DeleteSnatEndpointRules(snatClient *snat.SnatClient, hostToNC, NCToHost bool) { +func DeleteSnatEndpointRules(snatClient *snat.Client, hostToNC, ncToHost bool) { if hostToNC { - snatClient.DeleteInboundFromHostToNC() + err := snatClient.DeleteInboundFromHostToNC() + if err != nil { + log.Errorf("failed to delete inbound from host to nc rules") + } } - if NCToHost { - snatClient.DeleteInboundFromNCToHost() + if ncToHost { + err := snatClient.DeleteInboundFromNCToHost() + if err != nil { + log.Errorf("failed to delete inbound from nc to host rules") + } } } diff --git a/network/native_endpoint_snatroute_linux.go b/network/native_endpoint_snatroute_linux.go index d2eaa5bb16..c0d29eb435 100644 --- a/network/native_endpoint_snatroute_linux.go +++ b/network/native_endpoint_snatroute_linux.go @@ -6,10 +6,11 @@ import ( ) func (client *NativeEndpointClient) isSnatEnabled() bool { - return client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns + return client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDNS } -func (client *NativeEndpointClient) NewSnatClient(snatBridgeIP string, localIP string, epInfo *EndpointInfo) { - log.Printf("[native snat] %t %t %t %t", client.enableSnatOnHost, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.enableSnatForDns) + +func (client *NativeEndpointClient) NewSnatClient(snatBridgeIP, localIP string, epInfo *EndpointInfo) { + log.Printf("[native snat] %t %t %t %t", client.enableSnatOnHost, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.enableSnatForDNS) if client.isSnatEnabled() { client.snatClient = snat.NewSnatClient( GetSnatHostIfName(epInfo), @@ -23,6 +24,7 @@ func (client *NativeEndpointClient) NewSnatClient(snatBridgeIP string, localIP s ) } } + func (client *NativeEndpointClient) AddSnatEndpoint() error { if client.isSnatEnabled() { if err := AddSnatEndpoint(&client.snatClient); err != nil { diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 5d508e2ac2..1c03c82d03 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -43,13 +43,12 @@ type NativeEndpointClient struct { vnetNSName string vnetNSFileDescriptor int - snatClient snat.SnatClient + snatClient snat.Client vlanID int enableSnatOnHost bool - enableInfraVnet bool allowInboundFromHostToNC bool allowInboundFromNCToHost bool - enableSnatForDns bool + enableSnatForDNS bool netnsClient netnsClient netlink netlink.NetlinkInterface netioshim netio.NetIOInterface @@ -65,8 +64,8 @@ func NewNativeEndpointClient( vlanid int, localIP string, nl netlink.NetlinkInterface, - plc platform.ExecClient) *NativeEndpointClient { - + plc platform.ExecClient, +) *NativeEndpointClient { vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, vlanid) vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) @@ -83,7 +82,7 @@ func NewNativeEndpointClient( enableSnatOnHost: ep.EnableSnatOnHost, allowInboundFromHostToNC: ep.AllowInboundFromHostToNC, allowInboundFromNCToHost: ep.AllowInboundFromNCToHost, - enableSnatForDns: ep.EnableSnatForDns, + enableSnatForDNS: ep.EnableSnatForDns, netnsClient: netns.New(), netlink: nl, netioshim: &netio.NetIO{}, @@ -94,7 +93,6 @@ func NewNativeEndpointClient( client.NewSnatClient(nw.SnatBridgeIP, localIP, ep) return client - } // Adds interfaces to the vnet (created if not existing) and vm namespace @@ -389,7 +387,6 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) } func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { - if err := client.DeleteSnatEndpoint(); err != nil { return errors.Wrap(err, "failed to delete snat endpoint") } diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index d608fa72b4..e88cb062bb 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -7,6 +7,7 @@ import ( "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/snat" + "github.com/pkg/errors" ) // Communication between ovs switch and snat bridge, master of azuresnatveth0 is snat bridge itself @@ -18,7 +19,8 @@ const ( func (client *OVSEndpointClient) isSnatEnabled() bool { return client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns } -func (client *OVSEndpointClient) NewSnatClient(snatBridgeIP string, localIP string, epInfo *EndpointInfo) { + +func (client *OVSEndpointClient) NewSnatClient(snatBridgeIP, localIP string, epInfo *EndpointInfo) { if client.isSnatEnabled() { client.snatClient = snat.NewSnatClient( GetSnatHostIfName(epInfo), @@ -32,6 +34,7 @@ func (client *OVSEndpointClient) NewSnatClient(snatBridgeIP string, localIP stri ) } } + func (client *OVSEndpointClient) AddSnatEndpoint() error { if client.isSnatEnabled() { if err := AddSnatEndpoint(&client.snatClient); err != nil { @@ -66,7 +69,7 @@ func (client *OVSEndpointClient) AddSnatEndpoint() error { err = client.netlink.AddLink(&vethLink) if err != nil { log.Printf("[net] Failed to create veth pair, err:%v.", err) - return err + return errors.Wrap(err, "failed to create veth pair") } // Added in nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) @@ -81,20 +84,20 @@ func (client *OVSEndpointClient) AddSnatEndpoint() error { } // Separated - if err = client.netlink.SetLinkState(azureSnatVeth0, true); err != nil { - return err + if err := client.netlink.SetLinkState(azureSnatVeth0, true); err != nil { + return errors.Wrap(err, "failed to set azure snat veth 0 to up") } - if err = client.netlink.SetLinkMaster(azureSnatVeth0, snat.SnatBridgeName); err != nil { - return err + if err := client.netlink.SetLinkMaster(azureSnatVeth0, snat.SnatBridgeName); err != nil { + return errors.Wrap(err, "failed to set snat veth 0 master to snat bridge") } - if err = client.netlink.SetLinkState(azureSnatVeth1, true); err != nil { - return err + if err := client.netlink.SetLinkState(azureSnatVeth1, true); err != nil { + return errors.Wrap(err, "failed to set azure snat veth 1 to up") } if err := client.ovsctlClient.AddPortOnOVSBridge(azureSnatVeth1, client.bridgeName, 0); err != nil { - return err + return errors.Wrap(err, "failed to add port on OVS bridge") } } return nil diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index e89203e363..dd9300a146 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -23,7 +23,7 @@ type OVSEndpointClient struct { hostPrimaryMac string containerVethName string containerMac string - snatClient snat.SnatClient + snatClient snat.Client infraVnetClient ovsinfravnet.OVSInfraVnetClient vlanID int enableSnatOnHost bool @@ -51,8 +51,8 @@ func NewOVSEndpointClient( localIP string, nl netlink.NetlinkInterface, ovs ovsctl.OvsInterface, - plc platform.ExecClient) *OVSEndpointClient { - + plc platform.ExecClient, +) *OVSEndpointClient { client := &OVSEndpointClient{ bridgeName: nw.extIf.BridgeName, hostPrimaryIfName: nw.extIf.Name, @@ -241,6 +241,8 @@ func (client *OVSEndpointClient) DeleteEndpoints(ep *endpoint) error { return err } - client.DeleteSnatEndpoint() + if err := client.DeleteSnatEndpoint(); err != nil { + return err + } return DeleteInfraVnetEndpoint(client, ep.Id[:7]) } diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go index 76bd7b6151..da2b5c6bc2 100644 --- a/network/snat/snat_linux.go +++ b/network/snat/snat_linux.go @@ -33,7 +33,7 @@ func newErrorSnatClient(errStr string) error { return fmt.Errorf("%w : %s", errorSnatClient, errStr) } -type SnatClient struct { +type Client struct { hostSnatVethName string hostPrimaryMac string containerSnatVethName string @@ -54,9 +54,9 @@ func NewSnatClient(hostIfName string, nl netlink.NetlinkInterface, plClient platform.ExecClient, -) SnatClient { +) Client { log.Printf("Initialize new snat client") - snatClient := SnatClient{ + snatClient := Client{ hostSnatVethName: hostIfName, containerSnatVethName: contIfName, localIP: localIP, @@ -74,7 +74,7 @@ func NewSnatClient(hostIfName string, return snatClient } -func (client *SnatClient) CreateSnatEndpoint() error { +func (client *Client) CreateSnatEndpoint() error { // Create linux Bridge for outbound connectivity if err := client.createSnatBridge(client.SnatBridgeIP, client.hostPrimaryMac); err != nil { log.Printf("creating snat bridge failed with error %v", err) @@ -108,7 +108,7 @@ func (client *SnatClient) CreateSnatEndpoint() error { } // AllowIPAddressesOnSnatBridge adds iptables rules that allows only specific Private IPs via linux bridge -func (client *SnatClient) AllowIPAddressesOnSnatBridge() error { +func (client *Client) AllowIPAddressesOnSnatBridge() error { if err := networkutils.AllowIPAddresses(SnatBridgeName, client.SkipAddressesFromBlock, iptables.Insert); err != nil { log.Printf("AllowIPAddresses failed with error %v", err) return newErrorSnatClient(err.Error()) @@ -118,7 +118,7 @@ func (client *SnatClient) AllowIPAddressesOnSnatBridge() error { } // BlockIPAddressesOnSnatBridge adds iptables rules that blocks all private IPs flowing via linux bridge -func (client *SnatClient) BlockIPAddressesOnSnatBridge() error { +func (client *Client) BlockIPAddressesOnSnatBridge() error { if err := networkutils.BlockIPAddresses(SnatBridgeName, iptables.Append); err != nil { log.Printf("AllowIPAddresses failed with error %v", err) return newErrorSnatClient(err.Error()) @@ -130,7 +130,7 @@ func (client *SnatClient) BlockIPAddressesOnSnatBridge() error { /** Move container veth inside container network namespace **/ -func (client *SnatClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { +func (client *Client) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { log.Printf("[snat] Setting link %v netns %v.", client.containerSnatVethName, netnsPath) err := client.netlink.SetLinkNetNs(client.containerSnatVethName, nsID) if err != nil { @@ -142,7 +142,7 @@ func (client *SnatClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID u /** Configure Routes and setup name for container veth **/ -func (client *SnatClient) SetupSnatContainerInterface() error { +func (client *Client) SetupSnatContainerInterface() error { epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) if err := epc.SetupContainerInterface(client.containerSnatVethName, azureSnatIfName); err != nil { return newErrorSnatClient(err.Error()) @@ -153,7 +153,7 @@ func (client *SnatClient) SetupSnatContainerInterface() error { return nil } -func getNCLocalAndGatewayIP(client *SnatClient) (net.IP, net.IP) { +func getNCLocalAndGatewayIP(client *Client) (brIP, contIP net.IP) { bridgeIP, _, _ := net.ParseCIDR(client.SnatBridgeIP) containerIP, _, _ := net.ParseCIDR(client.localIP) return bridgeIP, containerIP @@ -162,7 +162,7 @@ func getNCLocalAndGatewayIP(client *SnatClient) (net.IP, net.IP) { /** This function adds iptables rules that allows only host to NC communication and not the other way **/ -func (client *SnatClient) AllowInboundFromHostToNC() error { +func (client *Client) AllowInboundFromHostToNC() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Create CNI Output chain @@ -218,7 +218,7 @@ func (client *SnatClient) AllowInboundFromHostToNC() error { return nil } -func (client *SnatClient) DeleteInboundFromHostToNC() error { +func (client *Client) DeleteInboundFromHostToNC() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Delete allow connection from Host to NC @@ -241,7 +241,7 @@ func (client *SnatClient) DeleteInboundFromHostToNC() error { /** This function adds iptables rules that allows only NC to Host communication and not the other way **/ -func (client *SnatClient) AllowInboundFromNCToHost() error { +func (client *Client) AllowInboundFromNCToHost() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Create CNI Input chain @@ -296,7 +296,7 @@ func (client *SnatClient) AllowInboundFromNCToHost() error { return err } -func (client *SnatClient) DeleteInboundFromNCToHost() error { +func (client *Client) DeleteInboundFromNCToHost() error { bridgeIP, containerIP := getNCLocalAndGatewayIP(client) // Delete allow NC to Host connection @@ -320,7 +320,7 @@ func (client *SnatClient) DeleteInboundFromNCToHost() error { Configures Local IP Address for container Veth **/ -func (client *SnatClient) ConfigureSnatContainerInterface() error { +func (client *Client) ConfigureSnatContainerInterface() error { log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.containerSnatVethName) ip, intIpAddr, _ := net.ParseCIDR(client.localIP) err := client.netlink.AddIPAddress(client.containerSnatVethName, ip, intIpAddr) @@ -330,7 +330,7 @@ func (client *SnatClient) ConfigureSnatContainerInterface() error { return nil } -func (client *SnatClient) DeleteSnatEndpoint() error { +func (client *Client) DeleteSnatEndpoint() error { log.Printf("[ovs] Deleting snat veth pair %v.", client.hostSnatVethName) err := client.netlink.DeleteLink(client.hostSnatVethName) if err != nil { @@ -341,7 +341,7 @@ func (client *SnatClient) DeleteSnatEndpoint() error { return nil } -func (client *SnatClient) setBridgeMac(hostPrimaryMac string) error { +func (client *Client) setBridgeMac(hostPrimaryMac string) error { hwAddr, err := net.ParseMAC(hostPrimaryMac) if err != nil { log.Errorf("Error while parsing host primary mac: %s error:%+v", hostPrimaryMac, err) @@ -354,7 +354,7 @@ func (client *SnatClient) setBridgeMac(hostPrimaryMac string) error { return err } -func (client *SnatClient) DropArpForSnatBridgeApipaRange(snatBridgeIP, azSnatVethIfName string) error { +func (client *Client) DropArpForSnatBridgeApipaRange(snatBridgeIP, azSnatVethIfName string) error { var err error _, ipCidr, _ := net.ParseCIDR(snatBridgeIP) if err = ebtables.SetArpDropRuleForIpCidr(ipCidr.String(), azSnatVethIfName); err != nil { @@ -367,7 +367,7 @@ func (client *SnatClient) DropArpForSnatBridgeApipaRange(snatBridgeIP, azSnatVet /** This function creates linux bridge which will be used for outbound connectivity by NCs **/ -func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac string) error { +func (client *Client) createSnatBridge(snatBridgeIP, hostPrimaryMac string) error { _, err := net.InterfaceByName(SnatBridgeName) if err == nil { log.Printf("Snat Bridge already exists") @@ -419,7 +419,7 @@ func (client *SnatClient) createSnatBridge(snatBridgeIP string, hostPrimaryMac s /** This function adds iptable rules that will snat all traffic that has source ip in apipa range and coming via linux bridge **/ -func (client *SnatClient) addMasqueradeRule(snatBridgeIPWithPrefix string) error { +func (client *Client) addMasqueradeRule(snatBridgeIPWithPrefix string) error { _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) matchCondition := fmt.Sprintf("-s %s", ipNet.String()) return iptables.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade) @@ -428,7 +428,7 @@ func (client *SnatClient) addMasqueradeRule(snatBridgeIPWithPrefix string) error /** Drop all vlan traffic on linux bridge **/ -func (client *SnatClient) addVlanDropRule() error { +func (client *Client) addVlanDropRule() error { out, err := client.plClient.ExecuteCommand(l2PreroutingEntries) if err != nil { log.Printf("Error while listing ebtable rules %v", err) diff --git a/network/snat/snat_linux_test.go b/network/snat/snat_linux_test.go index e0209c1201..1838b6aab1 100644 --- a/network/snat/snat_linux_test.go +++ b/network/snat/snat_linux_test.go @@ -19,7 +19,7 @@ func TestMain(m *testing.M) { func TestAllowInboundFromHostToNC(t *testing.T) { nl := netlink.NewNetlink() - client := &SnatClient{ + client := &Client{ SnatBridgeIP: "169.254.0.1/16", localIP: "169.254.0.4/16", containerSnatVethName: anyInterface, @@ -66,7 +66,7 @@ func TestAllowInboundFromHostToNC(t *testing.T) { func TestAllowInboundFromNCToHost(t *testing.T) { nl := netlink.NewNetlink() - client := &SnatClient{ + client := &Client{ SnatBridgeIP: "169.254.0.1/16", localIP: "169.254.0.4/16", containerSnatVethName: anyInterface, From c3e781f02b2708798989a8a49a31052122e9f129 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 27 Jul 2022 15:14:41 -0700 Subject: [PATCH 71/86] Go fumpt entire network package --- network/bridge_endpointclient_linux.go | 1 - network/endpoint_windows.go | 5 +---- network/endpoint_windows_test.go | 5 ----- network/hnsendpoint_mock_windows.go | 7 +++++-- network/hnswrapper/hnsv1wrapper.go | 6 +----- network/hnswrapper/hnsv1wrapperwithtimeout.go | 3 ++- network/hnswrapper/hnsv2wrapper.go | 3 +-- network/network_windows.go | 4 ++-- network/network_windows_test.go | 3 --- network/ovs_networkclient_linux.go | 3 ++- network/ovsinfravnet/infravnet_linux.go | 8 ++++---- network/transparent_endpointclient_linux.go | 1 - 12 files changed, 18 insertions(+), 31 deletions(-) diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go index 1150e37bfb..9d2ac09a6e 100644 --- a/network/bridge_endpointclient_linux.go +++ b/network/bridge_endpointclient_linux.go @@ -41,7 +41,6 @@ func NewLinuxBridgeEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, ) *LinuxBridgeEndpointClient { - client := &LinuxBridgeEndpointClient{ bridgeName: extIf.BridgeName, hostPrimaryIfName: extIf.Name, diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index ed25d3f4e4..0889e59d1d 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -64,7 +64,6 @@ func ConstructEndpointID(containerID string, netNsPath string, ifName string) (s // newEndpointImpl creates a new endpoint in the network. func (nw *network) newEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, _ platform.ExecClient, epInfo *EndpointInfo) (*endpoint, error) { - if useHnsV2, err := UseHnsV2(epInfo.NetNsPath); useHnsV2 { if err != nil { return nil, err @@ -115,7 +114,6 @@ func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo) (*endpoint, error) } hnsResponse, err := Hnsv1.CreateEndpoint(hnsEndpoint, "") - if err != nil { return nil, err } @@ -290,8 +288,7 @@ func (nw *network) createHostNCApipaEndpoint(cli apipaClient, epInfo *EndpointIn log.Printf("[net] Creating HostNCApipaEndpoint for host container connectivity for NC: %s", epInfo.NetworkContainerID) - if hostNCApipaEndpointID, err = - cli.CreateHostNCApipaEndpoint(context.TODO(), epInfo.NetworkContainerID); err != nil { + if hostNCApipaEndpointID, err = cli.CreateHostNCApipaEndpoint(context.TODO(), epInfo.NetworkContainerID); err != nil { return err } diff --git a/network/endpoint_windows_test.go b/network/endpoint_windows_test.go index 726cd85033..32bc79c8c9 100644 --- a/network/endpoint_windows_test.go +++ b/network/endpoint_windows_test.go @@ -40,7 +40,6 @@ func TestNewAndDeleteEndpointImplHnsV2(t *testing.T) { MacAddress: net.HardwareAddr("00:00:5e:00:53:01"), } endpoint, err := nw.newEndpointImplHnsV2(nil, epInfo) - if err != nil { fmt.Printf("+%v", err) t.Fatal(err) @@ -89,7 +88,6 @@ func TestNewEndpointImplHnsv2Timesout(t *testing.T) { if err == nil { t.Fatal("Failed to timeout HNS calls for creating endpoint") } - } func TestDeleteEndpointImplHnsv2Timeout(t *testing.T) { @@ -113,7 +111,6 @@ func TestDeleteEndpointImplHnsv2Timeout(t *testing.T) { MacAddress: net.HardwareAddr("00:00:5e:00:53:01"), } endpoint, err := nw.newEndpointImplHnsV2(nil, epInfo) - if err != nil { fmt.Printf("+%v", err) t.Fatal(err) @@ -167,7 +164,6 @@ func TestCreateEndpointImplHnsv1Timeout(t *testing.T) { if err == nil { t.Fatal("Failed to timeout HNS calls for creating endpoint") } - } func TestDeleteEndpointImplHnsv1Timeout(t *testing.T) { @@ -191,7 +187,6 @@ func TestDeleteEndpointImplHnsv1Timeout(t *testing.T) { MacAddress: net.HardwareAddr("00:00:5e:00:53:01"), } endpoint, err := nw.newEndpointImplHnsV1(epInfo) - if err != nil { fmt.Printf("+%v", err) t.Fatal(err) diff --git a/network/hnsendpoint_mock_windows.go b/network/hnsendpoint_mock_windows.go index 1a09cbcbb3..7556a76359 100644 --- a/network/hnsendpoint_mock_windows.go +++ b/network/hnsendpoint_mock_windows.go @@ -2,6 +2,7 @@ package network import ( "errors" + "github.com/Microsoft/hcsshim" ) @@ -12,9 +13,11 @@ type MockHNSEndpoint struct { } func NewMockHNSEndpoint(isAttached bool, hotAttachFailure bool) *MockHNSEndpoint { - return &MockHNSEndpoint{HnsIDMap: make(map[string]*hcsshim.HNSEndpoint), + return &MockHNSEndpoint{ + HnsIDMap: make(map[string]*hcsshim.HNSEndpoint), IsAttachedFlag: isAttached, - HotAttachFailure: hotAttachFailure} + HotAttachFailure: hotAttachFailure, + } } func (az *MockHNSEndpoint) GetHNSEndpointByName(endpointName string) (*hcsshim.HNSEndpoint, error) { diff --git a/network/hnswrapper/hnsv1wrapper.go b/network/hnswrapper/hnsv1wrapper.go index 55c117347f..cb5c39f8a2 100644 --- a/network/hnswrapper/hnsv1wrapper.go +++ b/network/hnswrapper/hnsv1wrapper.go @@ -10,8 +10,7 @@ import ( "github.com/Microsoft/hcsshim" ) -type Hnsv1wrapper struct { -} +type Hnsv1wrapper struct{} func (Hnsv1wrapper) CreateEndpoint(endpoint *hcsshim.HNSEndpoint, path string) (*hcsshim.HNSEndpoint, error) { // Marshal the request. @@ -35,13 +34,11 @@ func (Hnsv1wrapper) CreateEndpoint(endpoint *hcsshim.HNSEndpoint, path string) ( func (Hnsv1wrapper) DeleteEndpoint(endpointId string) (*hcsshim.HNSEndpoint, error) { hnsResponse, err := hcsshim.HNSEndpointRequest("DELETE", endpointId, "") - if err != nil { return nil, err } return hnsResponse, err - } func (Hnsv1wrapper) CreateNetwork(network *hcsshim.HNSNetwork, path string) (*hcsshim.HNSNetwork, error) { @@ -66,7 +63,6 @@ func (Hnsv1wrapper) CreateNetwork(network *hcsshim.HNSNetwork, path string) (*hc func (Hnsv1wrapper) DeleteNetwork(networkId string) (*hcsshim.HNSNetwork, error) { hnsResponse, err := hcsshim.HNSNetworkRequest("DELETE", networkId, "") - if err != nil { return nil, err } diff --git a/network/hnswrapper/hnsv1wrapperwithtimeout.go b/network/hnswrapper/hnsv1wrapperwithtimeout.go index d7c0ab1263..0b55a92b55 100644 --- a/network/hnswrapper/hnsv1wrapperwithtimeout.go +++ b/network/hnswrapper/hnsv1wrapperwithtimeout.go @@ -5,9 +5,10 @@ package hnswrapper import ( "context" + "time" + "github.com/Microsoft/hcsshim" "github.com/pkg/errors" - "time" ) type Hnsv1wrapperwithtimeout struct { diff --git a/network/hnswrapper/hnsv2wrapper.go b/network/hnswrapper/hnsv2wrapper.go index 7a5b9570a9..f375156948 100644 --- a/network/hnswrapper/hnsv2wrapper.go +++ b/network/hnswrapper/hnsv2wrapper.go @@ -10,8 +10,7 @@ import ( "github.com/Microsoft/hcsshim/hcn" ) -type Hnsv2wrapper struct { -} +type Hnsv2wrapper struct{} func (Hnsv2wrapper) CreateEndpoint(endpoint *hcn.HostComputeEndpoint) (*hcn.HostComputeEndpoint, error) { return endpoint.Create() diff --git a/network/network_windows.go b/network/network_windows.go index 09ff5b9ae5..b440e01842 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -65,14 +65,14 @@ var Hnsv1 hnswrapper.HnsV1WrapperInterface = hnswrapper.Hnsv1wrapper{} func EnableHnsV2Timeout(timeoutValue int) { if _, ok := Hnsv2.(hnswrapper.Hnsv2wrapperwithtimeout); !ok { - var timeoutDuration = time.Duration(timeoutValue) * time.Second + timeoutDuration := time.Duration(timeoutValue) * time.Second Hnsv2 = hnswrapper.Hnsv2wrapperwithtimeout{Hnsv2: hnswrapper.Hnsv2wrapper{}, HnsCallTimeout: timeoutDuration} } } func EnableHnsV1Timeout(timeoutValue int) { if _, ok := Hnsv1.(hnswrapper.Hnsv1wrapperwithtimeout); !ok { - var timeoutDuration = time.Duration(timeoutValue) * time.Second + timeoutDuration := time.Duration(timeoutValue) * time.Second Hnsv1 = hnswrapper.Hnsv1wrapperwithtimeout{Hnsv1: hnswrapper.Hnsv1wrapper{}, HnsCallTimeout: timeoutDuration} } } diff --git a/network/network_windows_test.go b/network/network_windows_test.go index 6eee00013c..4302ae402a 100644 --- a/network/network_windows_test.go +++ b/network/network_windows_test.go @@ -37,7 +37,6 @@ func TestNewAndDeleteNetworkImplHnsV2(t *testing.T) { } network, err := nm.newNetworkImplHnsV2(nwInfo, extInterface) - if err != nil { fmt.Printf("+%v", err) t.Fatal(err) @@ -137,7 +136,6 @@ func TestDeleteNetworkImplHnsV2WithTimeout(t *testing.T) { Hnsv2 = hnswrapper.NewHnsv2wrapperFake() network, err := nm.newNetworkImplHnsV2(nwInfo, extInterface) - if err != nil { fmt.Printf("+%v", err) t.Fatal(err) @@ -210,7 +208,6 @@ func TestDeleteNetworkImplHnsV1WithTimeout(t *testing.T) { Hnsv1 = hnswrapper.NewHnsv1wrapperFake() network, err := nm.newNetworkImplHnsV1(nwInfo, extInterface) - if err != nil { fmt.Printf("+%v", err) t.Fatal(err) diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index bd088b0678..3fdaf5afae 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -73,7 +73,8 @@ func (client *OVSNetworkClient) AddRoutes(nwInfo *NetworkInfo, interfaceName str } func NewOVSClient(bridgeName, hostInterfaceName string, ovsctlClient ovsctl.OvsInterface, - nl netlink.NetlinkInterface, plc platform.ExecClient) *OVSNetworkClient { + nl netlink.NetlinkInterface, plc platform.ExecClient, +) *OVSNetworkClient { ovsClient := &OVSNetworkClient{ bridgeName: bridgeName, hostInterfaceName: hostInterfaceName, diff --git a/network/ovsinfravnet/infravnet_linux.go b/network/ovsinfravnet/infravnet_linux.go index 0934718642..5a2e3ccc72 100644 --- a/network/ovsinfravnet/infravnet_linux.go +++ b/network/ovsinfravnet/infravnet_linux.go @@ -75,8 +75,8 @@ func (client *OVSInfraVnetClient) CreateInfraVnetRules( bridgeName string, infraIP net.IPNet, hostPrimaryMac string, - hostPort string) error { - + hostPort string, +) error { ovs := ovsctl.NewOvsctl() infraContainerPort, err := ovs.GetOVSPortNumber(client.hostInfraVethName) @@ -132,8 +132,8 @@ func (client *OVSInfraVnetClient) ConfigureInfraVnetContainerInterface(infraIP n func (client *OVSInfraVnetClient) DeleteInfraVnetRules( bridgeName string, infraIP net.IPNet, - hostPort string) { - + hostPort string, +) { ovs := ovsctl.NewOvsctl() log.Printf("[ovs] Deleting MAC DNAT rule for infravnet IP address %v", infraIP.IP.String()) diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 17a6d5677e..5364443a48 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -53,7 +53,6 @@ func NewTransparentEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, ) *TransparentEndpointClient { - client := &TransparentEndpointClient{ bridgeName: extIf.BridgeName, hostPrimaryIfName: extIf.Name, From 0fa38bf7513d0aae1b341df05715d6843c4bc30b Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Sun, 31 Jul 2022 22:10:22 -0700 Subject: [PATCH 72/86] Code markers removed, clean (Tests ok) OVS & Native: - Ping between two containers same VM, no packets on bridge - Ping between two containers diff VM, no packets on bridge - Ping other container not in vnet, no packets on bridge - Ping snat to container, packets on bridge - Ping container to snat, packets on bridge - Tcpdump confirmed on azSnatBr - Deletion of containers deletes appropriate interfaces --- network/native_endpoint_snatroute_linux.go | 2 -- network/ovs_endpoint_snatroute_linux.go | 5 +---- network/snat/snat_linux.go | 11 ++++------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/network/native_endpoint_snatroute_linux.go b/network/native_endpoint_snatroute_linux.go index c0d29eb435..39f1afcf02 100644 --- a/network/native_endpoint_snatroute_linux.go +++ b/network/native_endpoint_snatroute_linux.go @@ -1,7 +1,6 @@ package network import ( - "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/snat" ) @@ -10,7 +9,6 @@ func (client *NativeEndpointClient) isSnatEnabled() bool { } func (client *NativeEndpointClient) NewSnatClient(snatBridgeIP, localIP string, epInfo *EndpointInfo) { - log.Printf("[native snat] %t %t %t %t", client.enableSnatOnHost, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.enableSnatForDNS) if client.isSnatEnabled() { client.snatClient = snat.NewSnatClient( GetSnatHostIfName(epInfo), diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index e88cb062bb..a21f15c73d 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -40,10 +40,10 @@ func (client *OVSEndpointClient) AddSnatEndpoint() error { if err := AddSnatEndpoint(&client.snatClient); err != nil { return err } + // A lot of this code was in createSnatBridge initially and moved here since it is for ovs only snatClient := client.snatClient - // Separated log.Printf("Drop ARP for snat bridge ip: %s", snatClient.SnatBridgeIP) if err := client.snatClient.DropArpForSnatBridgeApipaRange(snatClient.SnatBridgeIP, azureSnatVeth0); err != nil { return err @@ -57,7 +57,6 @@ func (client *OVSEndpointClient) AddSnatEndpoint() error { return nil } - // Separated vethLink := netlink.VEthLink{ LinkInfo: netlink.LinkInfo{ Type: netlink.LINK_TYPE_VETH, @@ -71,7 +70,6 @@ func (client *OVSEndpointClient) AddSnatEndpoint() error { log.Printf("[net] Failed to create veth pair, err:%v.", err) return errors.Wrap(err, "failed to create veth pair") } - // Added in nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) //nolint if err = nuc.DisableRAForInterface(azureSnatVeth0); err != nil { @@ -83,7 +81,6 @@ func (client *OVSEndpointClient) AddSnatEndpoint() error { return err } - // Separated if err := client.netlink.SetLinkState(azureSnatVeth0, true); err != nil { return errors.Wrap(err, "failed to set azure snat veth 0 to up") } diff --git a/network/snat/snat_linux.go b/network/snat/snat_linux.go index da2b5c6bc2..b11009baa3 100644 --- a/network/snat/snat_linux.go +++ b/network/snat/snat_linux.go @@ -27,7 +27,7 @@ const ( l2PreroutingEntries = "ebtables -t nat -L PREROUTING" ) -var errorSnatClient = errors.New("OVSSnatClient Error") +var errorSnatClient = errors.New("SnatClient Error") func newErrorSnatClient(errStr string) error { return fmt.Errorf("%w : %s", errorSnatClient, errStr) @@ -321,7 +321,7 @@ func (client *Client) DeleteInboundFromNCToHost() error { **/ func (client *Client) ConfigureSnatContainerInterface() error { - log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.containerSnatVethName) + log.Printf("[snat] Adding IP address %v to link %v.", client.localIP, client.containerSnatVethName) ip, intIpAddr, _ := net.ParseCIDR(client.localIP) err := client.netlink.AddIPAddress(client.containerSnatVethName, ip, intIpAddr) if err != nil { @@ -331,10 +331,10 @@ func (client *Client) ConfigureSnatContainerInterface() error { } func (client *Client) DeleteSnatEndpoint() error { - log.Printf("[ovs] Deleting snat veth pair %v.", client.hostSnatVethName) + log.Printf("[snat] Deleting snat veth pair %v.", client.hostSnatVethName) err := client.netlink.DeleteLink(client.hostSnatVethName) if err != nil { - log.Printf("[ovs] Failed to delete veth pair %v: %v.", client.hostSnatVethName, err) + log.Printf("[snat] Failed to delete veth pair %v: %v.", client.hostSnatVethName, err) return newErrorSnatClient(err.Error()) } @@ -390,14 +390,12 @@ func (client *Client) createSnatBridge(snatBridgeIP, hostPrimaryMac string) erro if err := client.setBridgeMac(hostPrimaryMac); err != nil { return err } - // Drop ARP and InterfaceByName were here nuc := networkutils.NewNetworkUtils(client.netlink, client.plClient) //nolint if err = nuc.DisableRAForInterface(SnatBridgeName); err != nil { return err } - // VethLink, AddLink, and Disable RA for azureSnatVeth0/1 were here log.Printf("Assigning %v on snat bridge", snatBridgeIP) @@ -411,7 +409,6 @@ func (client *Client) createSnatBridge(snatBridgeIP, hostPrimaryMac string) erro if err = client.netlink.SetLinkState(SnatBridgeName, true); err != nil { return newErrorSnatClient(err.Error()) } - // Set link state of azure snatveth0/1 and link master of azuresnatveth0 set to snatbridgename return nil } From 18f9652786464398c531031980b16b94be790b1d Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 1 Aug 2022 11:02:32 -0700 Subject: [PATCH 73/86] Renamed veth, fixed logs --- network/endpoint_linux.go | 4 +-- network/native_endpointclient_linux.go | 24 ++++++------- network/native_endpointclient_linux_test.go | 38 ++++++++++----------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 9dbd21bfec..4c5330bd88 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -97,7 +97,7 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p epClient = &NativeEndpointClient{ eth0VethName: nw.extIf.Name, - vlanVethName: vlanVethName, + vlanEthName: vlanVethName, vnetVethName: hostIfName, containerVethName: contIfName, vnetNSName: vnetNSName, @@ -268,7 +268,7 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. epClient = &NativeEndpointClient{ eth0VethName: nw.extIf.Name, - vlanVethName: vlanVethName, + vlanEthName: vlanVethName, vnetVethName: ep.HostIfName, containerVethName: "", vnetNSName: vnetNSName, diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 2f1c637d88..ab1c1934c8 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -28,7 +28,7 @@ type netnsClient interface { } type NativeEndpointClient struct { eth0VethName string // So like eth0 - vlanVethName string // So like eth0.1 + vlanEthName string // So like eth0.1 vnetVethName string // Peer is containerVethName containerVethName string // Peer is vnetVethName @@ -102,14 +102,14 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { return errors.Wrap(deleteNSIfNotNilErr, "failed to get eth0 interface") } linkAttrs := vishnetlink.NewLinkAttrs() - linkAttrs.Name = client.vlanVethName + linkAttrs.Name = client.vlanEthName // Set the peer linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ LinkAttrs: linkAttrs, VlanId: client.vlanID, } - log.Printf("[native] Attempting to create %s link in VM NS", client.vlanVethName) + log.Printf("[native] Attempting to create %s link in VM NS", client.vlanEthName) // Create vlan veth deleteNSIfNotNilErr = vishnetlink.LinkAdd(link) if deleteNSIfNotNilErr != nil { @@ -117,16 +117,16 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { return errors.Wrap(deleteNSIfNotNilErr, "failed to create vlan vnet link after making new ns") } // vlan veth was created successfully, so move the vlan veth you created - log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) - deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) + log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanEthName, uintptr(client.vnetNSFileDescriptor)) + deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanEthName, uintptr(client.vnetNSFileDescriptor)) if deleteNSIfNotNilErr != nil { - if delErr := client.netlink.DeleteLink(client.vlanVethName); delErr != nil { + if delErr := client.netlink.DeleteLink(client.vlanEthName); delErr != nil { log.Errorf("deleting vlan veth failed on addendpoint failure") } return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") } } else { - log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanVethName) + log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanEthName) } client.vnetNSFileDescriptor = vnetNS @@ -151,7 +151,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Called from AddEndpoints, Namespace: Vnet func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { - _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanVethName) + _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanEthName) if err != nil { return errors.Wrap(err, "vlan veth doesn't exist") } @@ -241,10 +241,10 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo // Add route specifying which device the pod ip(s) are on routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) - if err = client.AddDefaultRoutes(client.vlanVethName); err != nil { + if err = client.AddDefaultRoutes(client.vlanEthName); err != nil { return errors.Wrap(err, "failed vnet ns add default/gateway routes (idempotent)") } - if err = client.AddDefaultArp(client.vlanVethName, azureMac); err != nil { + if err = client.AddDefaultArp(client.vlanEthName, azureMac); err != nil { return errors.Wrap(err, "failed vnet ns add default arp entry (idempotent)") } if err = addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { @@ -375,14 +375,14 @@ func ExecuteInNS(nsName string, f func() error) error { } defer ns.Close() // Enter the network namespace - log.Printf("[ExecuteInNS] Entering vnetns %s.", ns.file.Name()) + log.Printf("[ExecuteInNS] Entering ns %s.", ns.file.Name()) if err := ns.Enter(); err != nil { return err } // Exit network namespace defer func() { - log.Printf("[ExecuteInNS] Exiting vnetns %s.", ns.file.Name()) + log.Printf("[ExecuteInNS] Exiting ns %s.", ns.file.Name()) if err := ns.Exit(); err != nil { log.Errorf("[ExecuteInNS] Could not exit ns, err:%v.", err) } diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 615737e16b..cd7f199bd5 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -85,7 +85,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints create vnet ns failure", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -111,7 +111,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints with existing vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -134,7 +134,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints netlink fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -158,7 +158,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints get interface fail for primary interface (eth0)", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -184,7 +184,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints get interface fail for getting container veth", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -208,7 +208,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints NetNS Get fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -230,7 +230,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints NetNS Set fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -281,7 +281,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints get vnet veth mac address", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -297,7 +297,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints fail check vlan veth exists", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -314,7 +314,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints fail check vnet veth exists", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -368,7 +368,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { name: "Delete endpoint delete vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -390,7 +390,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { name: "Delete endpoint do not delete vnet ns it is still in use", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -414,7 +414,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { name: "Delete endpoint fail to delete namespace", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -468,7 +468,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes good path for container", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -492,7 +492,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes multiple IPs", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -524,7 +524,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes assign ip fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -549,7 +549,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes container 2nd default route added fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -595,7 +595,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes good path for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -620,7 +620,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes fail final routes for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", From 7e44c85368129b16406ed32b60d83e9a25f0a589 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 1 Aug 2022 11:38:41 -0700 Subject: [PATCH 74/86] Made deleteEndpoints logic clearer, renamed error --- network/native_endpointclient_linux.go | 28 ++++++++---- network/native_endpointclient_linux_test.go | 49 ++++++++++++--------- 2 files changed, 47 insertions(+), 30 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index ab1c1934c8..e1f60d2a74 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -327,24 +327,34 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { return ExecuteInNS(client.vnetNSName, func() error { - routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) - if err != nil { - return errors.Wrap(err, ("failed to get route list")) + // Passing in functionality to get number of routes after deletion + getNumRoutesLeft := func() (int, error) { + routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) + if err != nil { + return 0, errors.Wrap(err, "failed to get num routes left") + } + return len(routes), nil } - log.Printf("[native] Routes remaining: %v", routes) - return client.DeleteEndpointsImpl(ep, len(routes)) + return client.DeleteEndpointsImpl(ep, getNumRoutesLeft) }) } -func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint, routesLeft int) error { +// getNumRoutesLeft is a function which gets the current number of routes in the namespace. Namespace: Vnet +func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint, getNumRoutesLeft func() (int, error)) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) - log.Printf("[native] There are %d routes remaining", routesLeft) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return errors.Wrap(err, "failed to remove routes") } - // We get the # of routes, THEN delete, and see if it adds up - if routesLeft <= numDefaultRoutes+len(routeInfoList) { + + routesLeft, err := getNumRoutesLeft() + if err != nil { + return err + } + + log.Printf("[native] There are %d routes remaining after deletion", routesLeft) + + if routesLeft <= numDefaultRoutes { // Deletes default arp, default routes, vlan veth; there are two default routes // so when we have <= numDefaultRoutes routes left, no containers use this namespace log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index cd7f199bd5..8bb794b7d3 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -15,10 +15,10 @@ import ( "github.com/stretchr/testify/require" ) -var ErrorMock = errors.New("mock netns error") +var errNetnsMock = errors.New("mock netns error") -func newErrorMock(errStr string) error { - return errors.Wrap(ErrorMock, errStr) +func newNetnsErrorMock(errStr string) error { + return errors.Wrap(errNetnsMock, errStr) } type mockNetns struct { @@ -92,10 +92,10 @@ func TestNativeAddEndpoints(t *testing.T) { netnsClient: &mockNetns{ get: defaultGet, getFromName: func(name string) (fileDescriptor int, err error) { - return 0, newErrorMock("netns failure") + return 0, newNetnsErrorMock("netns failure") }, newNamed: func(name string) (fileDescriptor int, err error) { - return 0, newErrorMock("netns failure") + return 0, newNetnsErrorMock("netns failure") }, }, netlink: netlink.NewMockNetlink(false, ""), @@ -105,7 +105,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to create vnet ns: netns failure: " + ErrorMock.Error(), + wantErrMsg: "failed to create vnet ns: netns failure: " + errNetnsMock.Error(), }, { name: "Add endpoints with existing vnet ns", @@ -165,7 +165,7 @@ func TestNativeAddEndpoints(t *testing.T) { netnsClient: &mockNetns{ get: defaultGet, getFromName: func(name string) (fileDescriptor int, err error) { - return 0, newErrorMock("netns failure") + return 0, newNetnsErrorMock("netns failure") }, newNamed: defaultNewNamed, set: defaultSet, @@ -214,7 +214,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetNSName: "az_ns_1", netnsClient: &mockNetns{ get: func() (fileDescriptor int, err error) { - return 0, newErrorMock("netns failure") + return 0, newNetnsErrorMock("netns failure") }, }, netlink: netlink.NewMockNetlink(false, ""), @@ -224,7 +224,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to get vm ns handle: netns failure: " + ErrorMock.Error(), + wantErrMsg: "failed to get vm ns handle: netns failure: " + errNetnsMock.Error(), }, { name: "Add endpoints NetNS Set fail", @@ -237,11 +237,11 @@ func TestNativeAddEndpoints(t *testing.T) { netnsClient: &mockNetns{ get: defaultGet, getFromName: func(name string) (fileDescriptor int, err error) { - return 0, newErrorMock("do not fail on this error") + return 0, newNetnsErrorMock("do not fail on this error") }, newNamed: defaultNewNamed, set: func(fileDescriptor int) (err error) { - return newErrorMock("netns failure") + return newNetnsErrorMock("netns failure") }, deleteNamed: defaultDeleteNamed, }, @@ -252,7 +252,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to set current ns to vm: netns failure: " + ErrorMock.Error(), + wantErrMsg: "failed to set current ns to vm: netns failure: " + errNetnsMock.Error(), }, } @@ -356,13 +356,14 @@ func TestNativeDeleteEndpoints(t *testing.T) { Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), }, } + tests := []struct { name string client *NativeEndpointClient ep *endpoint wantErr bool wantErrMsg string - routesLeft int + routesLeft func() (int, error) }{ { name: "Delete endpoint delete vnet ns", @@ -383,8 +384,10 @@ func TestNativeDeleteEndpoints(t *testing.T) { ep: &endpoint{ IPAddresses: IPAddresses, }, - routesLeft: numDefaultRoutes + len(IPAddresses), - wantErr: false, + routesLeft: func() (int, error) { + return numDefaultRoutes, nil + }, + wantErr: false, }, { name: "Delete endpoint do not delete vnet ns it is still in use", @@ -396,7 +399,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetNSName: "az_ns_1", netnsClient: &mockNetns{ deleteNamed: func(name string) (err error) { - return newErrorMock("netns failure") + return newNetnsErrorMock("netns failure") }, }, netlink: netlink.NewMockNetlink(false, ""), @@ -407,8 +410,10 @@ func TestNativeDeleteEndpoints(t *testing.T) { ep: &endpoint{ IPAddresses: IPAddresses, }, - routesLeft: numDefaultRoutes + len(IPAddresses) + 1, - wantErr: false, + routesLeft: func() (int, error) { + return numDefaultRoutes + 1, nil + }, + wantErr: false, }, { name: "Delete endpoint fail to delete namespace", @@ -420,7 +425,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetNSName: "az_ns_1", netnsClient: &mockNetns{ deleteNamed: func(name string) (err error) { - return newErrorMock("netns failure") + return newNetnsErrorMock("netns failure") }, }, netlink: netlink.NewMockNetlink(false, ""), @@ -431,9 +436,11 @@ func TestNativeDeleteEndpoints(t *testing.T) { ep: &endpoint{ IPAddresses: IPAddresses, }, - routesLeft: numDefaultRoutes + len(IPAddresses), + routesLeft: func() (int, error) { + return numDefaultRoutes, nil + }, wantErr: true, - wantErrMsg: "failed to delete namespace: netns failure: " + ErrorMock.Error(), + wantErrMsg: "failed to delete namespace: netns failure: " + errNetnsMock.Error(), }, } From 18070940cf6a30f49bc76a5c89ddc3caaa6b3a9f Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 1 Aug 2022 14:54:23 -0700 Subject: [PATCH 75/86] Renamed eth0 to primaryHostIfName, vlanEth to vlanIf --- network/endpoint_linux.go | 8 +-- network/native_endpointclient_linux.go | 26 +++---- network/native_endpointclient_linux_test.go | 76 ++++++++++----------- 3 files changed, 55 insertions(+), 55 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 4c5330bd88..0c21b6c0d5 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -96,8 +96,8 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) epClient = &NativeEndpointClient{ - eth0VethName: nw.extIf.Name, - vlanEthName: vlanVethName, + primaryHostIfName: nw.extIf.Name, + vlanIfName: vlanVethName, vnetVethName: hostIfName, containerVethName: contIfName, vnetNSName: vnetNSName, @@ -267,8 +267,8 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. vnetNSName := fmt.Sprintf("az_ns_%d", ep.VlanID) epClient = &NativeEndpointClient{ - eth0VethName: nw.extIf.Name, - vlanEthName: vlanVethName, + primaryHostIfName: nw.extIf.Name, + vlanIfName: vlanVethName, vnetVethName: ep.HostIfName, containerVethName: "", vnetNSName: vnetNSName, diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index e1f60d2a74..8ddd4b6bec 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -27,8 +27,8 @@ type netnsClient interface { DeleteNamed(name string) (err error) } type NativeEndpointClient struct { - eth0VethName string // So like eth0 - vlanEthName string // So like eth0.1 + primaryHostIfName string // So like eth0 + vlanIfName string // So like eth0.1 vnetVethName string // Peer is containerVethName containerVethName string // Peer is vnetVethName @@ -95,21 +95,21 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { } // Now create vlan veth - log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) + log.Printf("[native] Create the host vlan link after getting eth0: %s", client.primaryHostIfName) // Get parent interface index. Index is consistent across libraries. - eth0, deleteNSIfNotNilErr := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) + eth0, deleteNSIfNotNilErr := client.netioshim.GetNetworkInterfaceByName(client.primaryHostIfName) if deleteNSIfNotNilErr != nil { return errors.Wrap(deleteNSIfNotNilErr, "failed to get eth0 interface") } linkAttrs := vishnetlink.NewLinkAttrs() - linkAttrs.Name = client.vlanEthName + linkAttrs.Name = client.vlanIfName // Set the peer linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ LinkAttrs: linkAttrs, VlanId: client.vlanID, } - log.Printf("[native] Attempting to create %s link in VM NS", client.vlanEthName) + log.Printf("[native] Attempting to create %s link in VM NS", client.vlanIfName) // Create vlan veth deleteNSIfNotNilErr = vishnetlink.LinkAdd(link) if deleteNSIfNotNilErr != nil { @@ -117,16 +117,16 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { return errors.Wrap(deleteNSIfNotNilErr, "failed to create vlan vnet link after making new ns") } // vlan veth was created successfully, so move the vlan veth you created - log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanEthName, uintptr(client.vnetNSFileDescriptor)) - deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanEthName, uintptr(client.vnetNSFileDescriptor)) + log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) + deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) if deleteNSIfNotNilErr != nil { - if delErr := client.netlink.DeleteLink(client.vlanEthName); delErr != nil { + if delErr := client.netlink.DeleteLink(client.vlanIfName); delErr != nil { log.Errorf("deleting vlan veth failed on addendpoint failure") } return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") } } else { - log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanEthName) + log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanIfName) } client.vnetNSFileDescriptor = vnetNS @@ -151,7 +151,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Called from AddEndpoints, Namespace: Vnet func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { - _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanEthName) + _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanIfName) if err != nil { return errors.Wrap(err, "vlan veth doesn't exist") } @@ -241,10 +241,10 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo // Add route specifying which device the pod ip(s) are on routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) - if err = client.AddDefaultRoutes(client.vlanEthName); err != nil { + if err = client.AddDefaultRoutes(client.vlanIfName); err != nil { return errors.Wrap(err, "failed vnet ns add default/gateway routes (idempotent)") } - if err = client.AddDefaultArp(client.vlanEthName, azureMac); err != nil { + if err = client.AddDefaultArp(client.vlanIfName, azureMac); err != nil { return errors.Wrap(err, "failed vnet ns add default arp entry (idempotent)") } if err = addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 8bb794b7d3..ab81ca6041 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -84,8 +84,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints create vnet ns failure", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -110,8 +110,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints with existing vnet ns", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -133,8 +133,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints netlink fail", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -157,8 +157,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints get interface fail for primary interface (eth0)", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -183,8 +183,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints get interface fail for getting container veth", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -207,8 +207,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints NetNS Get fail", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -229,8 +229,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints NetNS Set fail", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -280,8 +280,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints get vnet veth mac address", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -296,8 +296,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints fail check vlan veth exists", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -313,8 +313,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints fail check vnet veth exists", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -368,8 +368,8 @@ func TestNativeDeleteEndpoints(t *testing.T) { { name: "Delete endpoint delete vnet ns", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -392,8 +392,8 @@ func TestNativeDeleteEndpoints(t *testing.T) { { name: "Delete endpoint do not delete vnet ns it is still in use", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -418,8 +418,8 @@ func TestNativeDeleteEndpoints(t *testing.T) { { name: "Delete endpoint fail to delete namespace", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -474,8 +474,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { { name: "Configure interface and routes good path for container", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -498,8 +498,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { { name: "Configure interface and routes multiple IPs", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -530,8 +530,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { { name: "Configure interface and routes assign ip fail", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -555,8 +555,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { { name: "Configure interface and routes container 2nd default route added fail", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -601,8 +601,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { { name: "Configure interface and routes good path for vnet", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -626,8 +626,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { // fail route that tells which device container ip is on for vnet name: "Configure interface and routes fail final routes for vnet", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", From a99cb452a1a42328cd303468fcbb42fb8bd3abff Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 1 Aug 2022 15:17:49 -0700 Subject: [PATCH 76/86] Deleted debug log --- network/native_endpointclient_linux.go | 1 - 1 file changed, 1 deletion(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index bff8d4b4c1..8a95fd70b5 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -68,7 +68,6 @@ func NewNativeEndpointClient( vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, vlanid) vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) - log.Printf("[native snat constructor] %t %t %t %t", ep.EnableSnatOnHost, ep.AllowInboundFromHostToNC, ep.AllowInboundFromNCToHost, ep.EnableSnatForDns) client := &NativeEndpointClient{ primaryHostIfName: nw.extIf.Name, vlanIfName: vlanVethName, From 124c409422cea7d98c3e37dd7884608dd2a83ad4 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:31:21 -0700 Subject: [PATCH 77/86] Corrected merge (hardware addr) (Tests ok) --- network/native_endpointclient_linux.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index af203dfc88..78216f1361 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -34,8 +34,9 @@ type NativeEndpointClient struct { vnetVethName string // Peer is containerVethName containerVethName string // Peer is vnetVethName - vnetMac net.HardwareAddr - containerMac net.HardwareAddr + vnetMac net.HardwareAddr + containerMac net.HardwareAddr + hostPrimaryMac net.HardwareAddr vnetNSName string vnetNSFileDescriptor int From fed78649305ee174a8b155966ed7e563694a07ff Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 3 Aug 2022 17:02:05 -0700 Subject: [PATCH 78/86] Renamed vlan veth to hostExtIf_vlanID, Disabled RA eth0.2 makes disable RA look for a folder eth0 and then another sub folder "2". ("eth0/2") However, it should look for a folder named "eth0.2" literally. To solve this, we change the naming scheme to use an underscore instead. (Tests ok) --- network/native_endpointclient_linux.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 78216f1361..0cc601ab31 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -64,7 +64,7 @@ func NewNativeEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, ) *NativeEndpointClient { - vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, vlanid) + vlanVethName := fmt.Sprintf("%s_%d", nw.extIf.Name, vlanid) vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) client := &NativeEndpointClient{ @@ -163,6 +163,13 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Any failure to add the link should error (auto delete NS) return errors.Wrap(deleteNSIfNotNilErr, "failed to create vlan vnet link after making new ns") } + deleteNSIfNotNilErr = client.netUtilsClient.DisableRAForInterface(client.vlanIfName) + if deleteNSIfNotNilErr != nil { + if delErr := client.netlink.DeleteLink(client.vlanIfName); delErr != nil { + log.Errorf("deleting vlan veth failed on addendpoint failure") + } + return errors.Wrap(deleteNSIfNotNilErr, "failed to disable router advertisements for vlan vnet link") + } // vlan veth was created successfully, so move the vlan veth you created log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) @@ -180,6 +187,19 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { return errors.Wrap(err, "failed to create veth pair") } + // Disable RA for veth pair, and delete if any failure + if err = client.netUtilsClient.DisableRAForInterface(client.vnetVethName); err != nil { + if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { + log.Errorf("Deleting vnet veth failed on addendpoint failure:%v", delErr) + } + return errors.Wrap(err, "failed to disable RA on vnet veth, deleting") + } + if err = client.netUtilsClient.DisableRAForInterface(client.containerVethName); err != nil { + if delErr := client.netlink.DeleteLink(client.containerVethName); delErr != nil { + log.Errorf("Deleting container veth failed on addendpoint failure:%v", delErr) + } + return errors.Wrap(err, "failed to disable RA on container veth, deleting") + } if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { From 51879e28bb1f25578a22d8ddf9df9777c6f14a6a Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 8 Aug 2022 15:22:31 -0700 Subject: [PATCH 79/86] Renamed Native to TransparentVlan Confirmed basic functionality on VM with correct mode --- network/endpoint_linux.go | 12 ++-- network/network.go | 10 ++-- network/network_linux.go | 4 +- ...nsparent_vlan_endpoint_snatroute_linux.go} | 18 +++--- ... transparent_vlan_endpointclient_linux.go} | 58 +++++++++---------- ...sparent_vlan_endpointclient_linux_test.go} | 54 ++++++++--------- 6 files changed, 78 insertions(+), 78 deletions(-) rename network/{native_endpoint_snatroute_linux.go => transparent_vlan_endpoint_snatroute_linux.go} (66%) rename network/{native_endpointclient_linux.go => transparent_vlan_endpointclient_linux.go} (86%) rename network/{native_endpointclient_linux_test.go => transparent_vlan_endpointclient_linux_test.go} (94%) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index dab271bd28..00af9670ae 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -89,12 +89,12 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p } if vlanid != 0 { - if nw.Mode == opModeNative { - log.Printf("Native client") + if nw.Mode == opModeTransparentVlan { + log.Printf("Transparent vlan client") if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) } - epClient = NewNativeEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc) + epClient = NewTransparentVlanEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc) } else { log.Printf("OVS client") if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { @@ -247,9 +247,9 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. // entering the container netns and hence works both for CNI and CNM. if ep.VlanID != 0 { epInfo := ep.getInfo() - if nw.Mode == opModeNative { - log.Printf("Native client") - epClient = NewNativeEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, plc) + if nw.Mode == opModeTransparentVlan { + log.Printf("Transparent vlan client") + epClient = NewTransparentVlanEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, plc) } else { epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) diff --git a/network/network.go b/network/network.go index a28d0a7244..e520817f22 100644 --- a/network/network.go +++ b/network/network.go @@ -15,11 +15,11 @@ import ( const ( // Operational modes. - opModeBridge = "bridge" - opModeTunnel = "tunnel" - opModeTransparent = "transparent" - opModeNative = "native" - opModeDefault = opModeTunnel + opModeBridge = "bridge" + opModeTunnel = "tunnel" + opModeTransparent = "transparent" + opModeTransparentVlan = "transparentVlan" + opModeDefault = opModeTunnel ) const ( diff --git a/network/network_linux.go b/network/network_linux.go index 5ced57c8c2..50070e66ad 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -88,8 +88,8 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt return nil, fmt.Errorf("Ipv6 forwarding failed: %w", err) } } - case opModeNative: - log.Printf("Native mode") + case opModeTransparentVlan: + log.Printf("Transparent vlan mode") ifName = extIf.Name default: return nil, errNetworkModeInvalid diff --git a/network/native_endpoint_snatroute_linux.go b/network/transparent_vlan_endpoint_snatroute_linux.go similarity index 66% rename from network/native_endpoint_snatroute_linux.go rename to network/transparent_vlan_endpoint_snatroute_linux.go index 39f1afcf02..109f699b35 100644 --- a/network/native_endpoint_snatroute_linux.go +++ b/network/transparent_vlan_endpoint_snatroute_linux.go @@ -4,11 +4,11 @@ import ( "github.com/Azure/azure-container-networking/network/snat" ) -func (client *NativeEndpointClient) isSnatEnabled() bool { +func (client *TransparentVlanEndpointClient) isSnatEnabled() bool { return client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDNS } -func (client *NativeEndpointClient) NewSnatClient(snatBridgeIP, localIP string, epInfo *EndpointInfo) { +func (client *TransparentVlanEndpointClient) NewSnatClient(snatBridgeIP, localIP string, epInfo *EndpointInfo) { if client.isSnatEnabled() { client.snatClient = snat.NewSnatClient( GetSnatHostIfName(epInfo), @@ -23,7 +23,7 @@ func (client *NativeEndpointClient) NewSnatClient(snatBridgeIP, localIP string, } } -func (client *NativeEndpointClient) AddSnatEndpoint() error { +func (client *TransparentVlanEndpointClient) AddSnatEndpoint() error { if client.isSnatEnabled() { if err := AddSnatEndpoint(&client.snatClient); err != nil { return err @@ -32,7 +32,7 @@ func (client *NativeEndpointClient) AddSnatEndpoint() error { return nil } -func (client *NativeEndpointClient) AddSnatEndpointRules() error { +func (client *TransparentVlanEndpointClient) AddSnatEndpointRules() error { if client.isSnatEnabled() { // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge if err := AddSnatEndpointRules(&client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.netlink, client.plClient); err != nil { @@ -43,7 +43,7 @@ func (client *NativeEndpointClient) AddSnatEndpointRules() error { return nil } -func (client *NativeEndpointClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { +func (client *TransparentVlanEndpointClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { if client.isSnatEnabled() { return MoveSnatEndpointToContainerNS(&client.snatClient, netnsPath, nsID) } @@ -51,7 +51,7 @@ func (client *NativeEndpointClient) MoveSnatEndpointToContainerNS(netnsPath stri return nil } -func (client *NativeEndpointClient) SetupSnatContainerInterface() error { +func (client *TransparentVlanEndpointClient) SetupSnatContainerInterface() error { if client.isSnatEnabled() { return SetupSnatContainerInterface(&client.snatClient) } @@ -59,7 +59,7 @@ func (client *NativeEndpointClient) SetupSnatContainerInterface() error { return nil } -func (client *NativeEndpointClient) ConfigureSnatContainerInterface() error { +func (client *TransparentVlanEndpointClient) ConfigureSnatContainerInterface() error { if client.isSnatEnabled() { return ConfigureSnatContainerInterface(&client.snatClient) } @@ -67,7 +67,7 @@ func (client *NativeEndpointClient) ConfigureSnatContainerInterface() error { return nil } -func (client *NativeEndpointClient) DeleteSnatEndpoint() error { +func (client *TransparentVlanEndpointClient) DeleteSnatEndpoint() error { if client.isSnatEnabled() { return DeleteSnatEndpoint(&client.snatClient) } @@ -75,6 +75,6 @@ func (client *NativeEndpointClient) DeleteSnatEndpoint() error { return nil } -func (client *NativeEndpointClient) DeleteSnatEndpointRules() { +func (client *TransparentVlanEndpointClient) DeleteSnatEndpointRules() { DeleteSnatEndpointRules(&client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost) } diff --git a/network/native_endpointclient_linux.go b/network/transparent_vlan_endpointclient_linux.go similarity index 86% rename from network/native_endpointclient_linux.go rename to network/transparent_vlan_endpointclient_linux.go index 0cc601ab31..e62d85a559 100644 --- a/network/native_endpointclient_linux.go +++ b/network/transparent_vlan_endpointclient_linux.go @@ -28,7 +28,7 @@ type netnsClient interface { NewNamed(name string) (fileDescriptor int, err error) DeleteNamed(name string) (err error) } -type NativeEndpointClient struct { +type TransparentVlanEndpointClient struct { primaryHostIfName string // So like eth0 vlanIfName string // So like eth0.1 vnetVethName string // Peer is containerVethName @@ -54,7 +54,7 @@ type NativeEndpointClient struct { netUtilsClient networkutils.NetworkUtils } -func NewNativeEndpointClient( +func NewTransparentVlanEndpointClient( nw *network, ep *EndpointInfo, vnetVethName string, @@ -63,11 +63,11 @@ func NewNativeEndpointClient( localIP string, nl netlink.NetlinkInterface, plc platform.ExecClient, -) *NativeEndpointClient { +) *TransparentVlanEndpointClient { vlanVethName := fmt.Sprintf("%s_%d", nw.extIf.Name, vlanid) vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) - client := &NativeEndpointClient{ + client := &TransparentVlanEndpointClient{ primaryHostIfName: nw.extIf.Name, vlanIfName: vlanVethName, vnetVethName: vnetVethName, @@ -92,7 +92,7 @@ func NewNativeEndpointClient( } // Adds interfaces to the vnet (created if not existing) and vm namespace -func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { +func (client *TransparentVlanEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { // VM Namespace err := client.PopulateVM(epInfo) if err != nil { @@ -108,19 +108,19 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } // Called from AddEndpoints, Namespace: VM -func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { +func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) error { vmNS, err := client.netnsClient.Get() if err != nil { return errors.Wrap(err, "failed to get vm ns handle") } - log.Printf("[native] Checking if NS exists...") + log.Printf("[transparent vlan] Checking if NS exists...") vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) // If the ns does not exist, the below code will trigger to create it // This will also (we assume) mean the vlan veth does not exist if existingErr != nil { // We assume the only possible error is that the namespace doesn't exist - log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") + log.Printf("[transparent vlan] No existing NS detected. Creating the vnet namespace and switching to it") vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) if err != nil { return errors.Wrap(err, "failed to create vnet ns") @@ -130,7 +130,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Any failure will trigger removing the namespace created defer func() { if deleteNSIfNotNilErr != nil { - log.Logf("[native] Removing vnet ns due to failure...") + log.Logf("[transparent vlan] Removing vnet ns due to failure...") err = client.netnsClient.DeleteNamed(client.vnetNSName) if err != nil { log.Errorf("failed to cleanup/delete ns after failing to create vlan veth") @@ -142,7 +142,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { } // Now create vlan veth - log.Printf("[native] Create the host vlan link after getting eth0: %s", client.primaryHostIfName) + log.Printf("[transparent vlan] Create the host vlan link after getting eth0: %s", client.primaryHostIfName) // Get parent interface index. Index is consistent across libraries. eth0, deleteNSIfNotNilErr := client.netioshim.GetNetworkInterfaceByName(client.primaryHostIfName) if deleteNSIfNotNilErr != nil { @@ -156,7 +156,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { LinkAttrs: linkAttrs, VlanId: client.vlanID, } - log.Printf("[native] Attempting to create %s link in VM NS", client.vlanIfName) + log.Printf("[transparent vlan] Attempting to create %s link in VM NS", client.vlanIfName) // Create vlan veth deleteNSIfNotNilErr = vishnetlink.LinkAdd(link) if deleteNSIfNotNilErr != nil { @@ -171,7 +171,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { return errors.Wrap(deleteNSIfNotNilErr, "failed to disable router advertisements for vlan vnet link") } // vlan veth was created successfully, so move the vlan veth you created - log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) + log.Printf("[transparent vlan] Move vlan link (%s) to vnet NS: %d", client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) if deleteNSIfNotNilErr != nil { if delErr := client.netlink.DeleteLink(client.vlanIfName); delErr != nil { @@ -180,7 +180,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") } } else { - log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanIfName) + log.Printf("[transparent vlan] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanIfName) } client.vnetNSFileDescriptor = vnetNS @@ -217,7 +217,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { } // Called from AddEndpoints, Namespace: Vnet -func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { +func (client *TransparentVlanEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanIfName) if err != nil { return errors.Wrap(err, "vlan veth doesn't exist") @@ -230,18 +230,18 @@ func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { return nil } -func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { +func (client *TransparentVlanEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { if err := client.AddSnatEndpointRules(); err != nil { return errors.Wrap(err, "failed to add snat endpoint rules") } return nil } -func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { +func (client *TransparentVlanEndpointClient) DeleteEndpointRules(ep *endpoint) { client.DeleteSnatEndpointRules() } -func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { +func (client *TransparentVlanEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { return errors.Wrap(err, "failed to move endpoint to container ns") } @@ -251,7 +251,7 @@ func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointI return nil } -func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { +func (client *TransparentVlanEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return errors.Wrap(err, "failed to setup container interface") } @@ -264,7 +264,7 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf } // Adds routes, arp entries, etc. to the vnet and container namespaces -func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { +func (client *TransparentVlanEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { // Container NS err := client.ConfigureContainerInterfacesAndRoutesImpl(epInfo) if err != nil { @@ -282,7 +282,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo } // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Container -func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { +func (client *TransparentVlanEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return errors.Wrap(err, "failed to assign ips to container veth interface") } @@ -309,7 +309,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(ep } // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Vnet -func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { +func (client *TransparentVlanEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { return errors.Wrap(err, "failed to set loopback link state to up") @@ -333,7 +333,7 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo // Helper that gets the routes in the vnet NS for a particular list of IP addresses // Example: 192.168.0.4 dev proto static -func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []RouteInfo { +func (client *TransparentVlanEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []RouteInfo { routeInfoList := make([]RouteInfo, 0, len(ipAddresses)) // Add route specifying which device the pod ip(s) are on for _, ipAddr := range ipAddresses { @@ -347,7 +347,7 @@ func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []Rou } else { ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} } - log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) + log.Printf("[net] transparent vlan client adding route for the ip %v", ipNet.String()) routeInfo.Dst = ipNet routeInfoList = append(routeInfoList, routeInfo) @@ -359,7 +359,7 @@ func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []Rou // to the virtual gateway ip on linkToName device interface // Route 1: 169.254.1.1 dev // Route 2: default via 169.254.1.1 dev -func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { +func (client *TransparentVlanEndpointClient) AddDefaultRoutes(linkToName string) error { // Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0) virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ @@ -388,7 +388,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { // Helper that creates arp entry for the current NS which maps the virtual // gateway (169.254.1.1) to destMac on a particular interfaceName // Example: (169.254.1.1) at 12:34:56:78:9a:bc [ether] PERM on -func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) error { +func (client *TransparentVlanEndpointClient) AddDefaultArp(interfaceName, destMac string) error { _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", virtualGwNet.String(), destMac) @@ -402,7 +402,7 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) return nil } -func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { +func (client *TransparentVlanEndpointClient) DeleteEndpoints(ep *endpoint) error { if err := client.DeleteSnatEndpoint(); err != nil { return errors.Wrap(err, "failed to delete snat endpoint") } @@ -422,7 +422,7 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { } // getNumRoutesLeft is a function which gets the current number of routes in the namespace. Namespace: Vnet -func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint, getNumRoutesLeft func() (int, error)) error { +func (client *TransparentVlanEndpointClient) DeleteEndpointsImpl(ep *endpoint, getNumRoutesLeft func() (int, error)) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return errors.Wrap(err, "failed to remove routes") @@ -433,12 +433,12 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint, getNumRout return err } - log.Printf("[native] There are %d routes remaining after deletion", routesLeft) + log.Printf("[transparent vlan] There are %d routes remaining after deletion", routesLeft) if routesLeft <= numDefaultRoutes { // Deletes default arp, default routes, vlan veth; there are two default routes // so when we have <= numDefaultRoutes routes left, no containers use this namespace - log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) + log.Printf("[transparent vlan] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) if delErr != nil { return errors.Wrap(delErr, "failed to delete namespace") diff --git a/network/native_endpointclient_linux_test.go b/network/transparent_vlan_endpointclient_linux_test.go similarity index 94% rename from network/native_endpointclient_linux_test.go rename to network/transparent_vlan_endpointclient_linux_test.go index ab81ca6041..61fbf8cc78 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/transparent_vlan_endpointclient_linux_test.go @@ -69,13 +69,13 @@ func defaultDeleteNamed(name string) error { return nil } -func TestNativeAddEndpoints(t *testing.T) { +func TestTransparentVlanAddEndpoints(t *testing.T) { nl := netlink.NewMockNetlink(false, "") plc := platform.NewMockExecClient(false) tests := []struct { name string - client *NativeEndpointClient + client *TransparentVlanEndpointClient epInfo *EndpointInfo wantErr bool wantErrMsg string @@ -83,7 +83,7 @@ func TestNativeAddEndpoints(t *testing.T) { // Populating VM with data and creating interfaces/links { name: "Add endpoints create vnet ns failure", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -109,7 +109,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, { name: "Add endpoints with existing vnet ns", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -132,7 +132,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, { name: "Add endpoints netlink fail", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -156,7 +156,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, { name: "Add endpoints get interface fail for primary interface (eth0)", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -182,7 +182,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, { name: "Add endpoints get interface fail for getting container veth", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -206,7 +206,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, { name: "Add endpoints NetNS Get fail", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -228,7 +228,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, { name: "Add endpoints NetNS Set fail", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -271,7 +271,7 @@ func TestNativeAddEndpoints(t *testing.T) { tests = []struct { name string - client *NativeEndpointClient + client *TransparentVlanEndpointClient epInfo *EndpointInfo wantErr bool wantErrMsg string @@ -279,7 +279,7 @@ func TestNativeAddEndpoints(t *testing.T) { // Populate the client with information from the vnet and set up vnet { name: "Add endpoints get vnet veth mac address", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -295,7 +295,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, { name: "Add endpoints fail check vlan veth exists", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -312,7 +312,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, { name: "Add endpoints fail check vnet veth exists", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -343,7 +343,7 @@ func TestNativeAddEndpoints(t *testing.T) { } } -func TestNativeDeleteEndpoints(t *testing.T) { +func TestTransparentVlanDeleteEndpoints(t *testing.T) { nl := netlink.NewMockNetlink(false, "") plc := platform.NewMockExecClient(false) IPAddresses := []net.IPNet{ @@ -359,7 +359,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { tests := []struct { name string - client *NativeEndpointClient + client *TransparentVlanEndpointClient ep *endpoint wantErr bool wantErrMsg string @@ -367,7 +367,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { }{ { name: "Delete endpoint delete vnet ns", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -391,7 +391,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, { name: "Delete endpoint do not delete vnet ns it is still in use", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -417,7 +417,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, { name: "Delete endpoint fail to delete namespace", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -458,7 +458,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { } } -func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { +func TestTransparentVlanConfigureContainerInterfacesAndRoutes(t *testing.T) { nl := netlink.NewMockNetlink(false, "") plc := platform.NewMockExecClient(false) @@ -466,14 +466,14 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { tests := []struct { name string - client *NativeEndpointClient + client *TransparentVlanEndpointClient epInfo *EndpointInfo wantErr bool wantErrMsg string }{ { name: "Configure interface and routes good path for container", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -497,7 +497,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, { name: "Configure interface and routes multiple IPs", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -529,7 +529,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, { name: "Configure interface and routes assign ip fail", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -554,7 +554,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, { name: "Configure interface and routes container 2nd default route added fail", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -593,14 +593,14 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { } tests = []struct { name string - client *NativeEndpointClient + client *TransparentVlanEndpointClient epInfo *EndpointInfo wantErr bool wantErrMsg string }{ { name: "Configure interface and routes good path for vnet", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", @@ -625,7 +625,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { { // fail route that tells which device container ip is on for vnet name: "Configure interface and routes fail final routes for vnet", - client: &NativeEndpointClient{ + client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", vnetVethName: "A1veth0", From 5b187bc75af64188773d93b4c620a2729f6df50f Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 8 Aug 2022 16:00:50 -0700 Subject: [PATCH 80/86] Make file updated --- Makefile | 11 +++++++ ...nux-multitenancy-transparent-vlan.conflist | 29 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 cni/azure-linux-multitenancy-transparent-vlan.conflist diff --git a/Makefile b/Makefile index 946b459fd5..93823d4967 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ CNM_BUILD_DIR = $(BUILD_DIR)/cnm CNI_BUILD_DIR = $(BUILD_DIR)/cni ACNCLI_BUILD_DIR = $(BUILD_DIR)/acncli CNI_MULTITENANCY_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy +CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy-transparent-vlan CNI_SWIFT_BUILD_DIR = $(BUILD_DIR)/cni-swift CNI_OVERLAY_BUILD_DIR = $(BUILD_DIR)/cni-overlay CNI_BAREMETAL_BUILD_DIR = $(BUILD_DIR)/cni-baremetal @@ -77,6 +78,7 @@ CNM_ARCHIVE_NAME = azure-vnet-cnm-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) CNI_ARCHIVE_NAME = azure-vnet-cni-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) ACNCLI_ARCHIVE_NAME = acncli-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) CNI_MULTITENANCY_ARCHIVE_NAME = azure-vnet-cni-multitenancy-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) +CNI_MULTITENANCY_TRANSPARENT_VLAN_ARCHIVE_NAME = azure-vnet-cni-multitenancy-transparent-vlan-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) CNI_SWIFT_ARCHIVE_NAME = azure-vnet-cni-swift-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) CNI_OVERLAY_ARCHIVE_NAME = azure-vnet-cni-overlay-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) CNI_BAREMETAL_ARCHIVE_NAME = azure-vnet-cni-baremetal-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) @@ -439,6 +441,15 @@ ifeq ($(GOOS),linux) endif cd $(CNI_MULTITENANCY_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_MULTITENANCY_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config + $(MKDIR) $(CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR) + cp cni/azure-$(GOOS)-multitenancy-transparent-vlan.conflist $(CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR)/10-azure.conflist + cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR) +ifeq ($(GOOS),linux) + cp telemetry/azure-vnet-telemetry.config $(CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR)/azure-vnet-telemetry.config + cp $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR) +endif + cd $(CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_MULTITENANCY_TRANSPARENT_VLAN_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config + $(MKDIR) $(CNI_SWIFT_BUILD_DIR) cp cni/azure-$(GOOS)-swift.conflist $(CNI_SWIFT_BUILD_DIR)/10-azure.conflist cp telemetry/azure-vnet-telemetry.config $(CNI_SWIFT_BUILD_DIR)/azure-vnet-telemetry.config diff --git a/cni/azure-linux-multitenancy-transparent-vlan.conflist b/cni/azure-linux-multitenancy-transparent-vlan.conflist new file mode 100644 index 0000000000..7d2e20311a --- /dev/null +++ b/cni/azure-linux-multitenancy-transparent-vlan.conflist @@ -0,0 +1,29 @@ +{ + "cniVersion":"0.3.0", + "name":"azure", + "plugins":[ + { + "type":"azure-vnet", + "mode":"transparentVlan", + "bridge":"azure0", + "multiTenancy":true, + "infraVnetAddressSpace":"", + "podNamespaceForDualNetwork":[], + "enableExactMatchForPodName": false, + "enableSnatOnHost":true, + "ipam":{ + "type":"azure-cns" + }, + "dns":{ + "nameservers":[] + } + }, + { + "type":"portmap", + "capabilities":{ + "portMappings":true + }, + "snat":true + } + ] +} From c2594901040001bedbca767cdbc203ffd391f65a Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 8 Aug 2022 16:52:20 -0700 Subject: [PATCH 81/86] Create azure-windows-multitenancy-transparent-vlan.conflist --- ...ows-multitenancy-transparent-vlan.conflist | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 cni/azure-windows-multitenancy-transparent-vlan.conflist diff --git a/cni/azure-windows-multitenancy-transparent-vlan.conflist b/cni/azure-windows-multitenancy-transparent-vlan.conflist new file mode 100644 index 0000000000..d3a1b6e820 --- /dev/null +++ b/cni/azure-windows-multitenancy-transparent-vlan.conflist @@ -0,0 +1,52 @@ +{ + "cniVersion": "0.3.0", + "name": "azure", + "plugins": [ + { + "type": "azure-vnet", + "mode": "transparentVlan", + "bridge": "azure0", + "multiTenancy":true, + "enableSnatOnHost":true, + "enableExactMatchForPodName": true, + "capabilities": { + "portMappings": true + }, + "ipam": { + "type": "azure-cns" + }, + "dns": { + "Nameservers": [ + "10.0.0.10", + "168.63.129.16" + ], + "Search": [ + "svc.cluster.local" + ] + }, + "AdditionalArgs": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "ExceptionList": [ + "10.240.0.0/16", + "10.0.0.0/8" + ] + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ROUTE", + "DestinationPrefix": "10.0.0.0/8", + "NeedEncap": true + } + } + ], + "windowsSettings": { + "hnsTimeoutDurationInSeconds" : 120 + } + } + ] +} \ No newline at end of file From 21c26a4370ac3ca439fb542641690f4543154d26 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 9 Aug 2022 15:46:19 -0700 Subject: [PATCH 82/86] Unified snat err format --- network/endpoint_snatroute_linux.go | 29 ++++++++++++++++++++----- network/ovs_endpoint_snatroute_linux.go | 2 +- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/network/endpoint_snatroute_linux.go b/network/endpoint_snatroute_linux.go index a10c562b21..67f68d423a 100644 --- a/network/endpoint_snatroute_linux.go +++ b/network/endpoint_snatroute_linux.go @@ -20,7 +20,10 @@ func GetSnatContIfName(epInfo *EndpointInfo) string { } func AddSnatEndpoint(snatClient *snat.Client) error { - return errors.Wrap(snatClient.CreateSnatEndpoint(), "failed to add snat endpoint") + if err := snatClient.CreateSnatEndpoint(); err != nil { + return errors.Wrap(err, "failed to add snat endpoint") + } + return nil } func AddSnatEndpointRules(snatClient *snat.Client, hostToNC, ncToHost bool, nl netlink.NetlinkInterface, plc platform.ExecClient) error { @@ -45,25 +48,39 @@ func AddSnatEndpointRules(snatClient *snat.Client, hostToNC, ncToHost bool, nl n } if ncToHost { - return errors.Wrap(snatClient.AllowInboundFromNCToHost(), "failed to allow inbound from nc to host") + if err := snatClient.AllowInboundFromNCToHost(); err != nil { + return errors.Wrap(err, "failed to allow inbound from nc to host") + } } return nil } func MoveSnatEndpointToContainerNS(snatClient *snat.Client, netnsPath string, nsID uintptr) error { - return errors.Wrap(snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID), "failed to move snat endpoint to container ns") + if err := snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID); err != nil { + return errors.Wrap(err, "failed to move snat endpoint to container ns") + } + return nil } func SetupSnatContainerInterface(snatClient *snat.Client) error { - return errors.Wrap(snatClient.SetupSnatContainerInterface(), "failed to setup snat container interface") + if err := snatClient.SetupSnatContainerInterface(); err != nil { + return errors.Wrap(err, "failed to setup snat container interface") + } + return nil } func ConfigureSnatContainerInterface(snatClient *snat.Client) error { - return errors.Wrap(snatClient.ConfigureSnatContainerInterface(), "failed to configure snat container interface") + if err := snatClient.ConfigureSnatContainerInterface(); err != nil { + return errors.Wrap(err, "failed to configure snat container interface") + } + return nil } func DeleteSnatEndpoint(snatClient *snat.Client) error { - return errors.Wrap(snatClient.DeleteSnatEndpoint(), "failed to delete snat endpoint") + if err := snatClient.DeleteSnatEndpoint(); err != nil { + return errors.Wrap(err, "failed to delete snat endpoint") + } + return nil } func DeleteSnatEndpointRules(snatClient *snat.Client, hostToNC, ncToHost bool) { diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go index a21f15c73d..9f43b2a6a2 100644 --- a/network/ovs_endpoint_snatroute_linux.go +++ b/network/ovs_endpoint_snatroute_linux.go @@ -102,7 +102,7 @@ func (client *OVSEndpointClient) AddSnatEndpoint() error { func (client *OVSEndpointClient) AddSnatEndpointRules() error { if client.isSnatEnabled() { - // Add route for 169.254.169.54 in host via azure0, otherwise it will route via snat bridge + // Add route for 169.254.169.254 in host via azure0, otherwise it will route via snat bridge if err := AddSnatEndpointRules(&client.snatClient, client.allowInboundFromHostToNC, client.allowInboundFromNCToHost, client.netlink, client.plClient); err != nil { return err } From 1f18c1fd223570116ef108efcc0b5b520cdeb0c5 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 9 Aug 2022 15:47:17 -0700 Subject: [PATCH 83/86] Rename to transparent-vlan --- cni/azure-linux-multitenancy-transparent-vlan.conflist | 2 +- cni/azure-windows-multitenancy-transparent-vlan.conflist | 2 +- network/network.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cni/azure-linux-multitenancy-transparent-vlan.conflist b/cni/azure-linux-multitenancy-transparent-vlan.conflist index 7d2e20311a..a997d1981f 100644 --- a/cni/azure-linux-multitenancy-transparent-vlan.conflist +++ b/cni/azure-linux-multitenancy-transparent-vlan.conflist @@ -4,7 +4,7 @@ "plugins":[ { "type":"azure-vnet", - "mode":"transparentVlan", + "mode":"transparent-vlan", "bridge":"azure0", "multiTenancy":true, "infraVnetAddressSpace":"", diff --git a/cni/azure-windows-multitenancy-transparent-vlan.conflist b/cni/azure-windows-multitenancy-transparent-vlan.conflist index d3a1b6e820..21031acab3 100644 --- a/cni/azure-windows-multitenancy-transparent-vlan.conflist +++ b/cni/azure-windows-multitenancy-transparent-vlan.conflist @@ -4,7 +4,7 @@ "plugins": [ { "type": "azure-vnet", - "mode": "transparentVlan", + "mode": "transparent-vlan", "bridge": "azure0", "multiTenancy":true, "enableSnatOnHost":true, diff --git a/network/network.go b/network/network.go index e520817f22..bdcb80612b 100644 --- a/network/network.go +++ b/network/network.go @@ -18,7 +18,7 @@ const ( opModeBridge = "bridge" opModeTunnel = "tunnel" opModeTransparent = "transparent" - opModeTransparentVlan = "transparentVlan" + opModeTransparentVlan = "transparent-vlan" opModeDefault = opModeTunnel ) From dd57afe9c11f39c7aa56d76bb5fb98022ecdadd8 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 9 Aug 2022 15:47:46 -0700 Subject: [PATCH 84/86] Route table support added to local netlink --- network/endpoint.go | 1 + network/endpoint_linux.go | 1 + 2 files changed, 2 insertions(+) diff --git a/network/endpoint.go b/network/endpoint.go index ec51bea1bf..ae3d1008a9 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -93,6 +93,7 @@ type RouteInfo struct { DevName string Scope int Priority int + Table int } type apipaClient interface { diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 00af9670ae..92400ec8af 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -301,6 +301,7 @@ func addRoutes(nl netlink.NetlinkInterface, netioshim netio.NetIOInterface, inte Priority: route.Priority, Protocol: route.Protocol, Scope: route.Scope, + Table: route.Table, } if err := nl.AddIPRoute(nlRoute); err != nil { From b2674dc556642058913731cbab0d9d39c8a60924 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 9 Aug 2022 15:53:36 -0700 Subject: [PATCH 85/86] Moved SNAT to end of function --- .../transparent_vlan_endpointclient_linux.go | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/network/transparent_vlan_endpointclient_linux.go b/network/transparent_vlan_endpointclient_linux.go index e62d85a559..06e0e4db9e 100644 --- a/network/transparent_vlan_endpointclient_linux.go +++ b/network/transparent_vlan_endpointclient_linux.go @@ -271,14 +271,19 @@ func (client *TransparentVlanEndpointClient) ConfigureContainerInterfacesAndRout return err } - if err = client.ConfigureSnatContainerInterface(); err != nil { - return errors.Wrap(err, "failed to configure snat container interface") - } - // Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes - return ExecuteInNS(client.vnetNSName, func() error { + err = ExecuteInNS(client.vnetNSName, func() error { return client.ConfigureVnetInterfacesAndRoutesImpl(epInfo) }) + if err != nil { + return err + } + + // Container NS + if err = client.ConfigureSnatContainerInterface(); err != nil { + return errors.Wrap(err, "failed to configure snat container interface") + } + return nil } // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Container @@ -403,11 +408,8 @@ func (client *TransparentVlanEndpointClient) AddDefaultArp(interfaceName, destMa } func (client *TransparentVlanEndpointClient) DeleteEndpoints(ep *endpoint) error { - if err := client.DeleteSnatEndpoint(); err != nil { - return errors.Wrap(err, "failed to delete snat endpoint") - } - - return ExecuteInNS(client.vnetNSName, func() error { + // Vnet NS + err := ExecuteInNS(client.vnetNSName, func() error { // Passing in functionality to get number of routes after deletion getNumRoutesLeft := func() (int, error) { routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) @@ -419,6 +421,15 @@ func (client *TransparentVlanEndpointClient) DeleteEndpoints(ep *endpoint) error return client.DeleteEndpointsImpl(ep, getNumRoutesLeft) }) + if err != nil { + return err + } + + // VM NS + if err := client.DeleteSnatEndpoint(); err != nil { + return errors.Wrap(err, "failed to delete snat endpoint") + } + return nil } // getNumRoutesLeft is a function which gets the current number of routes in the namespace. Namespace: Vnet From 6f1c50a748729ac4fff7bffb9abfd3cb6ef710d1 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 9 Aug 2022 16:01:33 -0700 Subject: [PATCH 86/86] Defer deleting vlan interface on failure --- network/transparent_vlan_endpointclient_linux.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/network/transparent_vlan_endpointclient_linux.go b/network/transparent_vlan_endpointclient_linux.go index 06e0e4db9e..ba75d5f1ee 100644 --- a/network/transparent_vlan_endpointclient_linux.go +++ b/network/transparent_vlan_endpointclient_linux.go @@ -130,7 +130,7 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er // Any failure will trigger removing the namespace created defer func() { if deleteNSIfNotNilErr != nil { - log.Logf("[transparent vlan] Removing vnet ns due to failure...") + log.Logf("[transparent vlan] removing vnet ns due to failure...") err = client.netnsClient.DeleteNamed(client.vnetNSName) if err != nil { log.Errorf("failed to cleanup/delete ns after failing to create vlan veth") @@ -163,20 +163,22 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er // Any failure to add the link should error (auto delete NS) return errors.Wrap(deleteNSIfNotNilErr, "failed to create vlan vnet link after making new ns") } + defer func() { + if deleteNSIfNotNilErr != nil { + log.Logf("[transparent vlan] removing vlan veth due to failure...") + if delErr := client.netlink.DeleteLink(client.vlanIfName); delErr != nil { + log.Errorf("deleting vlan veth failed on addendpoint failure") + } + } + }() deleteNSIfNotNilErr = client.netUtilsClient.DisableRAForInterface(client.vlanIfName) if deleteNSIfNotNilErr != nil { - if delErr := client.netlink.DeleteLink(client.vlanIfName); delErr != nil { - log.Errorf("deleting vlan veth failed on addendpoint failure") - } return errors.Wrap(deleteNSIfNotNilErr, "failed to disable router advertisements for vlan vnet link") } // vlan veth was created successfully, so move the vlan veth you created log.Printf("[transparent vlan] Move vlan link (%s) to vnet NS: %d", client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) if deleteNSIfNotNilErr != nil { - if delErr := client.netlink.DeleteLink(client.vlanIfName); delErr != nil { - log.Errorf("deleting vlan veth failed on addendpoint failure") - } return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") } } else {