From b877895596dffc99221cf46a1afeecaeacab49cb Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 5 Jul 2022 17:01:47 -0700 Subject: [PATCH 01/52] Native Endpoint Client Add Endpoints --- go.mod | 3 +- go.sum | 4 + network/endpoint_linux.go | 48 ++++-- network/native_endpointclient_linux.go | 220 +++++++++++++++++++++++++ network/network.go | 1 + network/network_linux.go | 3 + 6 files changed, 264 insertions(+), 15 deletions(-) create mode 100644 network/native_endpointclient_linux.go diff --git a/go.mod b/go.mod index 715dccb9b6..44a75cbde0 100644 --- a/go.mod +++ b/go.mod @@ -104,7 +104,8 @@ require ( github.com/subosito/gotenv v1.3.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect - github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect + github.com/vishvananda/netlink v1.2.1-beta.2 + github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.6.0 // indirect diff --git a/go.sum b/go.sum index 9ad21a839e..60a2b412a3 100644 --- a/go.sum +++ b/go.sum @@ -832,11 +832,15 @@ github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= +github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg= +github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index b084895097..f983f502ea 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -58,6 +58,8 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p var localIP string var epClient EndpointClient var vlanid int = 0 + var vnetIfName string + var vnetNSName string if nw.Endpoints[epInfo.Id] != nil { log.Printf("[net] Endpoint alreday exists.") @@ -89,21 +91,39 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p } if vlanid != 0 { - log.Printf("OVS client") - if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { - nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) - } + if nw.Mode == opModeNative { + log.Printf("Mode %s", nw.Mode) + log.Printf("Native client") + vnetIfName = fmt.Sprintf("eth0.%s", vlanid) + vnetNSName = fmt.Sprintf("az_ns_%s", vlanid) + + epClient = NewNativeEndpointClient( + nw.extIf, + hostIfName, + vnetIfName, + contIfName, + vnetNSName, + nw.Mode, + vlanid, + nl, + plc) + } else { + log.Printf("OVS client") + if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { + nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) + } - epClient = NewOVSEndpointClient( - nw, - epInfo, - hostIfName, - contIfName, - vlanid, - localIP, - nl, - ovsctl.NewOvsctl(), - plc) + epClient = NewOVSEndpointClient( + nw, + epInfo, + hostIfName, + contIfName, + vlanid, + localIP, + nl, + ovsctl.NewOvsctl(), + plc) + } } else if nw.Mode != opModeTransparent { log.Printf("Bridge client") epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go new file mode 100644 index 0000000000..cb6f4c88b4 --- /dev/null +++ b/network/native_endpointclient_linux.go @@ -0,0 +1,220 @@ +package network + +import ( + "errors" + "fmt" + "net" + + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/platform" + + vishnetlink "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" +) + +var errorNativeEndpointClient = errors.New("TransparentEndpointClient Error") + +func newErrorNativeEndpointClient(errStr string) error { + return fmt.Errorf("%w : %s", errorTransparentEndpointClient, errStr) +} + +type NativeEndpointClient struct { + hostPrimaryIfName string //So like eth0 + hostVethName string //So like eth0.X + vnetVethName string //Peer is containerVethName + containerVethName string //Peer is vnetVethName + + hostPrimaryMac net.HardwareAddr + vnetMac net.HardwareAddr + containerMac net.HardwareAddr + hostVethMac net.HardwareAddr + + vnetNSName string + vnetNS *Namespace + dns *DNSInfo + + mode string + vlanID int + netlink netlink.NetlinkInterface + netioshim netio.NetIOInterface + plClient platform.ExecClient + netUtilsClient networkutils.NetworkUtils +} + +func NewNativeEndpointClient( + extIf *externalInterface, + hostVethName string, + vnetVethName string, + containerVethName string, + vnetNSName string, + mode string, + vlanid int, + nl netlink.NetlinkInterface, + plc platform.ExecClient, +) *NativeEndpointClient { + + client := &NativeEndpointClient{ + hostPrimaryIfName: extIf.Name, + hostVethName: hostVethName, + vnetVethName: vnetVethName, + containerVethName: containerVethName, + hostPrimaryMac: extIf.MacAddress, + vnetNSName: vnetNSName, + mode: mode, + vlanID: vlanid, + netlink: nl, + netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + } + + return client +} + +func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { + var err error + //Create the vnet namespace + if _, err = netns.NewNamed(client.vnetNSName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + //Create said namespace (doesn't enter or exit yet) + vnetNS, err := OpenNamespace(client.vnetNSName) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + client.vnetNS = vnetNS + + //Create the host vlan link and move it to vnet NS + linkAttrs := vishnetlink.NewLinkAttrs() + linkAttrs.Name = client.hostVethName + //TODO: linkAttrs.ParentIndex = eth0 (Primary) + link := &vishnetlink.Vlan{ + LinkAttrs: linkAttrs, + VlanId: client.vlanID, + } + vishnetlink.LinkAdd(link) + if err = client.netlink.SetLinkNetNs(client.hostVethName, client.vnetNS.GetFd()); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + //Create veth pair (automatically set to UP) + if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + //Move vnetVethName into vnet namespace (peer will be moved in MoveEndpointsToContainerNS) + if err = client.netlink.SetLinkNetNs(client.vnetVethName, client.vnetNS.GetFd()); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + //If there is a failure, delete the links + defer func() { + if err != nil { + //Delete vnet <> container + if delErr := client.netlink.DeleteLink(client.hostVethName); delErr != nil { + log.Errorf("Deleting veth failed on addendpoint failure:%v", delErr) + } + //Delete eth0 <> eth0.X + + } + }() + //Check that container veth exists. DOES it matter if they are in a different NS? + containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + client.containerMac = containerIf.HardwareAddr + + //Check that host veth exists (eth0.X) + hostVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + client.hostVethMac = hostVethIf.HardwareAddr + + //Check that vnet veth exists + vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + client.vnetMac = vnetVethIf.HardwareAddr + //Set MTU? + + return nil +} +func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { + //Described as rules on ip addresses on the container interface, + + // ip route add dev + // This route is needed for incoming packets to pod to route via hostveth + //Transparent has routes going to host, but we don't need that here, right? + //Each endpoint only has two routes, which are the default. + + //What is arp proxy? Set up arp rules here? + //AddEndpointRules vs. ConfigureContainerInterfacesAndRoutes + return nil +} +func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { + return nil +} +func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { + if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + return nil +} +func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { + if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + client.containerVethName = epInfo.IfName + + return nil +} +func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + // add route for virtualgwip + // ip route add 169.254.1.1/32 dev eth0, but where do you specify eth0? + // addRoutes says it adds route to containerVethName, so this appears to be a route to the container? + + virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) + routeInfo := RouteInfo{ + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + } + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + return newErrorTransparentEndpointClient(err.Error()) + } + + // ip route add default via 169.254.1.1 dev eth0 + _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) + dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} + routeInfo = RouteInfo{ + Dst: dstIP, + Gw: virtualGwIP, + } + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + return err + } + + // arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac + log.Printf("[net] Adding static arp for IP address %v and MAC %v in Container namespace", + virtualGwNet.String(), client.hostVethMac) + if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, + client.containerVethName, + virtualGwNet.IP, + client.hostVethMac, + false); err != nil { + return fmt.Errorf("Adding arp in container failed: %w", err) + } + + return nil +} +func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { + return nil +} diff --git a/network/network.go b/network/network.go index 16979d7414..a28d0a7244 100644 --- a/network/network.go +++ b/network/network.go @@ -18,6 +18,7 @@ const ( opModeBridge = "bridge" opModeTunnel = "tunnel" opModeTransparent = "transparent" + opModeNative = "native" opModeDefault = opModeTunnel ) diff --git a/network/network_linux.go b/network/network_linux.go index 2afb4a5a0f..5ced57c8c2 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -88,6 +88,9 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt return nil, fmt.Errorf("Ipv6 forwarding failed: %w", err) } } + case opModeNative: + log.Printf("Native mode") + ifName = extIf.Name default: return nil, errNetworkModeInvalid } From 7f4ae0c92a0505365443522d87d1671ec54e1c43 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 6 Jul 2022 10:44:42 -0700 Subject: [PATCH 02/52] AddEndpointRules, ConfigureContainerInterfacesAndRoutes --- network/endpoint_linux.go | 4 +- network/native_endpointclient_linux.go | 167 +++++++++++++++++++------ 2 files changed, 130 insertions(+), 41 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index f983f502ea..4d8eb08a0e 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -94,8 +94,8 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p if nw.Mode == opModeNative { log.Printf("Mode %s", nw.Mode) log.Printf("Native client") - vnetIfName = fmt.Sprintf("eth0.%s", vlanid) - vnetNSName = fmt.Sprintf("az_ns_%s", vlanid) + vnetIfName = fmt.Sprintf("eth0.%d", vlanid) + vnetNSName = fmt.Sprintf("az_ns_%d", vlanid) epClient = NewNativeEndpointClient( nw.extIf, diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index cb6f4c88b4..7cf7da9353 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -15,10 +15,14 @@ import ( "github.com/vishvananda/netns" ) -var errorNativeEndpointClient = errors.New("TransparentEndpointClient Error") +const ( + azureMac = "12:34:56:78:9a:bc" +) + +var errorNativeEndpointClient = errors.New("NativeEndpointClient Error") func newErrorNativeEndpointClient(errStr string) error { - return fmt.Errorf("%w : %s", errorTransparentEndpointClient, errStr) + return fmt.Errorf("%w : %s", errorNativeEndpointClient, errStr) } type NativeEndpointClient struct { @@ -33,7 +37,6 @@ type NativeEndpointClient struct { hostVethMac net.HardwareAddr vnetNSName string - vnetNS *Namespace dns *DNSInfo mode string @@ -85,39 +88,45 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if err != nil { return newErrorNativeEndpointClient(err.Error()) } - client.vnetNS = vnetNS + defer vnetNS.Close() //Create the host vlan link and move it to vnet NS linkAttrs := vishnetlink.NewLinkAttrs() linkAttrs.Name = client.hostVethName - //TODO: linkAttrs.ParentIndex = eth0 (Primary) + //Get parent interface index. Is the index the same regardless of lib used? + eth0, err := client.netioshim.GetNetworkInterfaceByName(client.hostPrimaryIfName) + //Is this how you set the peer? + linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ LinkAttrs: linkAttrs, VlanId: client.vlanID, } vishnetlink.LinkAdd(link) - if err = client.netlink.SetLinkNetNs(client.hostVethName, client.vnetNS.GetFd()); err != nil { + if err = client.netlink.SetLinkNetNs(client.hostVethName, vnetNS.GetFd()); err != nil { return newErrorNativeEndpointClient(err.Error()) } + vishnetlink.LinkSetUp(link) //Create veth pair (automatically set to UP) if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } //Move vnetVethName into vnet namespace (peer will be moved in MoveEndpointsToContainerNS) - if err = client.netlink.SetLinkNetNs(client.vnetVethName, client.vnetNS.GetFd()); err != nil { + if err = client.netlink.SetLinkNetNs(client.vnetVethName, vnetNS.GetFd()); err != nil { return newErrorNativeEndpointClient(err.Error()) } //If there is a failure, delete the links defer func() { if err != nil { - //Delete vnet <> container + //Delete vnet <-> container + if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { + log.Errorf("Deleting vnetVeth failed on addendpoint failure:%v", delErr) + } + //Delete eth0 <-> eth0.X if delErr := client.netlink.DeleteLink(client.hostVethName); delErr != nil { - log.Errorf("Deleting veth failed on addendpoint failure:%v", delErr) + log.Errorf("Deleting hostVeth failed on addendpoint failure:%v", delErr) } - //Delete eth0 <> eth0.X - } }() //Check that container veth exists. DOES it matter if they are in a different NS? @@ -154,41 +163,77 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error //What is arp proxy? Set up arp rules here? //AddEndpointRules vs. ConfigureContainerInterfacesAndRoutes - return nil -} -func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { - return nil -} -func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { - if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { + + //Is it okay if I enter the namespace and then exit and enter again? + //Is it better just to call OpenNamespace every time? + + // Open the vnet network namespace + log.Printf("Opening vnetns %v.", client.vnetNSName) + ns, err := OpenNamespace(client.vnetNSName) + if err != nil { + return err + } + defer ns.Close() + // Enter the vnet network namespace + log.Printf("Entering vnetns %v.", ns) + if err := ns.Enter(); err != nil { + return err + } + + // Exit vnet network namespace + defer func() { + if err := ns.Exit(); err != nil { + log.Printf("Could not exit vnetns, err:%v.", err) + } + }() + + var routeInfoList []RouteInfo + + // ip route add dev + // This route is needed for incoming packets to pod to route via hostveth + for _, ipAddr := range epInfo.IPAddresses { + var ( + routeInfo RouteInfo + ipNet net.IPNet + ) + + if ipAddr.IP.To4() != nil { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} + } else { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} + } + log.Printf("[net] Adding route for the ip %v", ipNet.String()) + routeInfo.Dst = ipNet + routeInfoList = append(routeInfoList, routeInfo) + + } + if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return newErrorNativeEndpointClient(err.Error()) } - return nil -} -func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + // Add default/gateway routes (Assuming indempotent) + if err = client.AddDefaultRoutes(client.hostVethName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + // Add ARP entry (Assuming indempotent) + if err = client.AddDefaultArp(client.vnetVethName, azureMac); err != nil { return newErrorNativeEndpointClient(err.Error()) } - client.containerVethName = epInfo.IfName return nil } -func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } +// Helper that creates routing rules for the current NS which direct packets +// to the virtual gateway ip on linkToName device interface +func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { // add route for virtualgwip - // ip route add 169.254.1.1/32 dev eth0, but where do you specify eth0? - // addRoutes says it adds route to containerVethName, so this appears to be a route to the container? - + // ip route add 169.254.1.1/32 dev eth0 virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ Dst: *virtualGwNet, Scope: netlink.RT_SCOPE_LINK, } - if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { - return newErrorTransparentEndpointClient(err.Error()) + if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { + return err } // ip route add default via 169.254.1.1 dev eth0 @@ -198,19 +243,63 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo Dst: dstIP, Gw: virtualGwIP, } - if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { return err } + return nil +} - // arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac - log.Printf("[net] Adding static arp for IP address %v and MAC %v in Container namespace", - virtualGwNet.String(), client.hostVethMac) +// Helper that creates arp entry for the current NS which maps the virtual +// gateway to destMac +func (client *NativeEndpointClient) AddDefaultArp(name string, destMac string) error { + _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) + // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac + log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", + virtualGwNet.String(), destMac) + hardwareAddr, err := net.ParseMAC(destMac) + if err != nil { + return err + } if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, - client.containerVethName, + name, //What is the purpose of name? virtualGwNet.IP, - client.hostVethMac, + hardwareAddr, false); err != nil { - return fmt.Errorf("Adding arp in container failed: %w", err) + return fmt.Errorf("adding arp entry failed: %w", err) + } + return nil +} +func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { + +} +func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { + if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + return nil +} +func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { + if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + client.containerVethName = epInfo.IfName + + return nil +} +func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + // add route for virtualgwip + // ip route add 169.254.1.1/32 dev eth0, but where do you specify eth0? + // addRoutes says it adds route to containerVethName, so this appears to be a route to the container? + //Currently called from container NS + if err := client.AddDefaultRoutes(client.containerVethName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + if err := client.AddDefaultArp(epInfo.NetNsPath, client.vnetMac.String()); err != nil { + return newErrorNativeEndpointClient(err.Error()) } return nil From 89b5f277e81b52fb10af26b7b1883e36beee1737 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 6 Jul 2022 12:31:46 -0700 Subject: [PATCH 03/52] Changed interface names, log statements nw.extIf.Name > eth0 (eth0) eth0.vlanid > eth0.X (eth0.1) %s%s hostIfName > vnet (A1veth0) %s%s-2 contIfName > container (B1veth0) --- network/endpoint_linux.go | 10 +-- network/native_endpointclient_linux.go | 103 +++++++++++++++---------- 2 files changed, 67 insertions(+), 46 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 4d8eb08a0e..c59ee0a813 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -58,7 +58,7 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p var localIP string var epClient EndpointClient var vlanid int = 0 - var vnetIfName string + var ethXIfName string var vnetNSName string if nw.Endpoints[epInfo.Id] != nil { @@ -94,13 +94,13 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p if nw.Mode == opModeNative { log.Printf("Mode %s", nw.Mode) log.Printf("Native client") - vnetIfName = fmt.Sprintf("eth0.%d", vlanid) + ethXIfName = fmt.Sprintf("eth0.%d", vlanid) vnetNSName = fmt.Sprintf("az_ns_%d", vlanid) - + //hostIfName may be a misnomer as this end is in the vnet NS epClient = NewNativeEndpointClient( - nw.extIf, + nw.extIf.Name, + ethXIfName, hostIfName, - vnetIfName, contIfName, vnetNSName, nw.Mode, diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 7cf7da9353..8b4773293c 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net" + "os" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" @@ -26,18 +27,17 @@ func newErrorNativeEndpointClient(errStr string) error { } type NativeEndpointClient struct { - hostPrimaryIfName string //So like eth0 - hostVethName string //So like eth0.X + eth0VethName string //So like eth0 + ethXVethName string //So like eth0.X vnetVethName string //Peer is containerVethName containerVethName string //Peer is vnetVethName - hostPrimaryMac net.HardwareAddr - vnetMac net.HardwareAddr - containerMac net.HardwareAddr - hostVethMac net.HardwareAddr + //hostPrimaryMac net.HardwareAddr + vnetMac net.HardwareAddr + containerMac net.HardwareAddr + hostVethMac net.HardwareAddr vnetNSName string - dns *DNSInfo mode string vlanID int @@ -48,8 +48,8 @@ type NativeEndpointClient struct { } func NewNativeEndpointClient( - extIf *externalInterface, - hostVethName string, + eth0VethName string, + ethXVethName string, vnetVethName string, containerVethName string, vnetNSName string, @@ -58,20 +58,21 @@ func NewNativeEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, ) *NativeEndpointClient { - + log.Printf("Create new native client: eth0:%s,ethX:%s,vnet:%s,cont:%s,id:%s", + eth0VethName, ethXVethName, vnetVethName, containerVethName, vlanid) client := &NativeEndpointClient{ - hostPrimaryIfName: extIf.Name, - hostVethName: hostVethName, + eth0VethName: eth0VethName, + ethXVethName: ethXVethName, vnetVethName: vnetVethName, containerVethName: containerVethName, - hostPrimaryMac: extIf.MacAddress, - vnetNSName: vnetNSName, - mode: mode, - vlanID: vlanid, - netlink: nl, - netioshim: &netio.NetIO{}, - plClient: plc, - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + //hostPrimaryMac: extIf.MacAddress, + vnetNSName: vnetNSName, + mode: mode, + vlanID: vlanid, + netlink: nl, + netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), } return client @@ -79,22 +80,22 @@ func NewNativeEndpointClient( func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { var err error - //Create the vnet namespace + log.Printf("Create the vnet namespace") if _, err = netns.NewNamed(client.vnetNSName); err != nil { return newErrorNativeEndpointClient(err.Error()) } - //Create said namespace (doesn't enter or exit yet) + log.Printf("Open said namespace and get NS reference(doesn't enter or exit yet)") vnetNS, err := OpenNamespace(client.vnetNSName) if err != nil { return newErrorNativeEndpointClient(err.Error()) } defer vnetNS.Close() - //Create the host vlan link and move it to vnet NS + log.Printf("Create the host vlan link") linkAttrs := vishnetlink.NewLinkAttrs() - linkAttrs.Name = client.hostVethName + linkAttrs.Name = client.ethXVethName //Get parent interface index. Is the index the same regardless of lib used? - eth0, err := client.netioshim.GetNetworkInterfaceByName(client.hostPrimaryIfName) + eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) //Is this how you set the peer? linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ @@ -102,16 +103,17 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { VlanId: client.vlanID, } vishnetlink.LinkAdd(link) - if err = client.netlink.SetLinkNetNs(client.hostVethName, vnetNS.GetFd()); err != nil { + log.Printf("Move vlan link to vnet NS") + if err = client.netlink.SetLinkNetNs(client.ethXVethName, vnetNS.GetFd()); err != nil { return newErrorNativeEndpointClient(err.Error()) } vishnetlink.LinkSetUp(link) - //Create veth pair (automatically set to UP) + log.Printf("Create veth pair (automatically set to UP)") if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } - //Move vnetVethName into vnet namespace (peer will be moved in MoveEndpointsToContainerNS) + log.Printf("Move vnetVethName into vnet namespace (peer will be moved in MoveEndpointsToContainerNS)") if err = client.netlink.SetLinkNetNs(client.vnetVethName, vnetNS.GetFd()); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -119,32 +121,33 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { //If there is a failure, delete the links defer func() { if err != nil { + log.Printf("Failure detected, deleting links...") //Delete vnet <-> container if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { log.Errorf("Deleting vnetVeth failed on addendpoint failure:%v", delErr) } //Delete eth0 <-> eth0.X - if delErr := client.netlink.DeleteLink(client.hostVethName); delErr != nil { + if delErr := client.netlink.DeleteLink(client.ethXVethName); delErr != nil { log.Errorf("Deleting hostVeth failed on addendpoint failure:%v", delErr) } } }() - //Check that container veth exists. DOES it matter if they are in a different NS? + log.Printf("Check that container veth exists.") containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) if err != nil { return newErrorNativeEndpointClient(err.Error()) } client.containerMac = containerIf.HardwareAddr - //Check that host veth exists (eth0.X) - hostVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName) + log.Printf("Check that (eth0.X) exists") + hostVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) if err != nil { return newErrorNativeEndpointClient(err.Error()) } client.hostVethMac = hostVethIf.HardwareAddr - //Check that vnet veth exists - vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName) + log.Printf("Check that vnet veth exists") + vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) if err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -182,6 +185,7 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error // Exit vnet network namespace defer func() { + log.Printf("Exiting vnetns %v.", ns) if err := ns.Exit(); err != nil { log.Printf("Could not exit vnetns, err:%v.", err) } @@ -202,7 +206,7 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error } else { ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} } - log.Printf("[net] Adding route for the ip %v", ipNet.String()) + log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) routeInfo.Dst = ipNet routeInfoList = append(routeInfoList, routeInfo) @@ -210,11 +214,11 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return newErrorNativeEndpointClient(err.Error()) } - // Add default/gateway routes (Assuming indempotent) - if err = client.AddDefaultRoutes(client.hostVethName); err != nil { + log.Printf("Vnet NS add default/gateway routes (Assuming indempotent)") + if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } - // Add ARP entry (Assuming indempotent) + log.Printf("Vnet NS add default ARP entry (Assuming indempotent)") if err = client.AddDefaultArp(client.vnetVethName, azureMac); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -225,7 +229,7 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error // Helper that creates routing rules for the current NS which direct packets // to the virtual gateway ip on linkToName device interface func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { - // add route for virtualgwip + log.Printf("Add route for virtualgwip") // ip route add 169.254.1.1/32 dev eth0 virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ @@ -236,7 +240,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { return err } - // ip route add default via 169.254.1.1 dev eth0 + log.Printf("Add default route (ip route add default via 169.254.1.1 dev eth0)") _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} routeInfo = RouteInfo{ @@ -273,12 +277,14 @@ func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { + log.Printf("Moving endpoint to container NS") if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { return newErrorNativeEndpointClient(err.Error()) } return nil } func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { + log.Printf("Setup container interface") if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -287,21 +293,36 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf return nil } func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + log.Printf("Assign IPs to container veth interface") if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorNativeEndpointClient(err.Error()) } - // add route for virtualgwip + log.Printf("Container NS add route for virtualgwip") // ip route add 169.254.1.1/32 dev eth0, but where do you specify eth0? // addRoutes says it adds route to containerVethName, so this appears to be a route to the container? //Currently called from container NS if err := client.AddDefaultRoutes(client.containerVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } + log.Printf("Container NS add arp entry") if err := client.AddDefaultArp(epInfo.NetNsPath, client.vnetMac.String()); err != nil { return newErrorNativeEndpointClient(err.Error()) } + log.Printf("Create resolv.conf for DNS") + folder := fmt.Sprintf("/etc/netns/%s", client.vnetNSName) + resolv := fmt.Sprintf("%s/resolv.conf", folder) + if _, err := os.Stat(folder); os.IsNotExist(err) { + os.MkdirAll(folder, 0700) // Create your file + } + log.Printf("Writing to resolv.conf file %s , %s", epInfo.DNS.Servers, epInfo.DNS.Servers[0]) + data := []byte(fmt.Sprintf("nameserver %s", epInfo.DNS.Servers[0])) + err := os.WriteFile(resolv, data, 0644) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + return nil } func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { From abb2bca27f7b2868ce3ee6765dc0e46e19e89b28 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 7 Jul 2022 16:44:33 -0700 Subject: [PATCH 04/52] Renaming, using lib to set ns --- network/native_endpointclient_linux.go | 218 +++++++++++++++---------- 1 file changed, 128 insertions(+), 90 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 8b4773293c..440d901034 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "net" - "os" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" @@ -35,9 +34,12 @@ type NativeEndpointClient struct { //hostPrimaryMac net.HardwareAddr vnetMac net.HardwareAddr containerMac net.HardwareAddr - hostVethMac net.HardwareAddr + ethXMac net.HardwareAddr + + vmNS netns.NsHandle vnetNSName string + vnetNS netns.NsHandle mode string vlanID int @@ -80,31 +82,42 @@ func NewNativeEndpointClient( func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { var err error - log.Printf("Create the vnet namespace") - if _, err = netns.NewNamed(client.vnetNSName); err != nil { + log.Printf("Get VM namespace handle") + vmNS, err := netns.Get() + if err != nil { return newErrorNativeEndpointClient(err.Error()) } - log.Printf("Open said namespace and get NS reference(doesn't enter or exit yet)") - vnetNS, err := OpenNamespace(client.vnetNSName) - if err != nil { + log.Printf("Save VM namespace: %s", vmNS) + client.vmNS = vmNS + + log.Printf("Create the vnet namespace") + if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { return newErrorNativeEndpointClient(err.Error()) } - defer vnetNS.Close() + log.Printf("Set current namespace to VM: %s", vmNS) + netns.Set(vmNS) - log.Printf("Create the host vlan link") + log.Printf("Create the host vlan link after getting %s", client.eth0VethName) linkAttrs := vishnetlink.NewLinkAttrs() linkAttrs.Name = client.ethXVethName //Get parent interface index. Is the index the same regardless of lib used? eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) + if err != nil { + log.Printf("Failed to get interface: %s", client.eth0VethName) + return newErrorNativeEndpointClient(err.Error()) + } //Is this how you set the peer? linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ LinkAttrs: linkAttrs, VlanId: client.vlanID, } - vishnetlink.LinkAdd(link) - log.Printf("Move vlan link to vnet NS") - if err = client.netlink.SetLinkNetNs(client.ethXVethName, vnetNS.GetFd()); err != nil { + err = vishnetlink.LinkAdd(link) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + log.Printf("Move vlan link to vnet NS: %d", uintptr(client.vnetNS)) + if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNS)); err != nil { return newErrorNativeEndpointClient(err.Error()) } vishnetlink.LinkSetUp(link) @@ -114,13 +127,15 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return newErrorNativeEndpointClient(err.Error()) } log.Printf("Move vnetVethName into vnet namespace (peer will be moved in MoveEndpointsToContainerNS)") - if err = client.netlink.SetLinkNetNs(client.vnetVethName, vnetNS.GetFd()); err != nil { + if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNS)); err != nil { return newErrorNativeEndpointClient(err.Error()) } //If there is a failure, delete the links defer func() { if err != nil { + log.Printf("Switching NS to vnet") + netns.Set(client.vnetNS) log.Printf("Failure detected, deleting links...") //Delete vnet <-> container if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { @@ -130,6 +145,8 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if delErr := client.netlink.DeleteLink(client.ethXVethName); delErr != nil { log.Errorf("Deleting hostVeth failed on addendpoint failure:%v", delErr) } + log.Printf("Switching NS to vm") + netns.Set(client.vmNS) } }() log.Printf("Check that container veth exists.") @@ -139,12 +156,21 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } client.containerMac = containerIf.HardwareAddr + log.Printf("Switch NS to vnet") + netns.Set(client.vnetNS) + + currNS, err := netns.Get() + log.Printf("Current NS after switch to vnet: %v.", currNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + log.Printf("Check that (eth0.X) exists") - hostVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) + ethXVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) if err != nil { return newErrorNativeEndpointClient(err.Error()) } - client.hostVethMac = hostVethIf.HardwareAddr + client.ethXMac = ethXVethIf.HardwareAddr log.Printf("Check that vnet veth exists") vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) @@ -153,75 +179,14 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } client.vnetMac = vnetVethIf.HardwareAddr //Set MTU? + log.Printf("Switch NS to vm") + netns.Set(client.vmNS) return nil } func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { - //Described as rules on ip addresses on the container interface, - - // ip route add dev - // This route is needed for incoming packets to pod to route via hostveth - //Transparent has routes going to host, but we don't need that here, right? - //Each endpoint only has two routes, which are the default. - - //What is arp proxy? Set up arp rules here? - //AddEndpointRules vs. ConfigureContainerInterfacesAndRoutes - - //Is it okay if I enter the namespace and then exit and enter again? - //Is it better just to call OpenNamespace every time? - - // Open the vnet network namespace - log.Printf("Opening vnetns %v.", client.vnetNSName) - ns, err := OpenNamespace(client.vnetNSName) - if err != nil { - return err - } - defer ns.Close() - // Enter the vnet network namespace - log.Printf("Entering vnetns %v.", ns) - if err := ns.Enter(); err != nil { - return err - } - - // Exit vnet network namespace - defer func() { - log.Printf("Exiting vnetns %v.", ns) - if err := ns.Exit(); err != nil { - log.Printf("Could not exit vnetns, err:%v.", err) - } - }() - - var routeInfoList []RouteInfo - - // ip route add dev - // This route is needed for incoming packets to pod to route via hostveth - for _, ipAddr := range epInfo.IPAddresses { - var ( - routeInfo RouteInfo - ipNet net.IPNet - ) - - if ipAddr.IP.To4() != nil { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} - } else { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} - } - log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) - routeInfo.Dst = ipNet - routeInfoList = append(routeInfoList, routeInfo) - - } - if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - log.Printf("Vnet NS add default/gateway routes (Assuming indempotent)") - if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - log.Printf("Vnet NS add default ARP entry (Assuming indempotent)") - if err = client.AddDefaultArp(client.vnetVethName, azureMac); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } + //There are no rules to add here + //Described as rules on ip addresses on the container interface return nil } @@ -233,8 +198,9 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { // ip route add 169.254.1.1/32 dev eth0 virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ - Dst: *virtualGwNet, - Scope: netlink.RT_SCOPE_LINK, + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + DevName: linkToName, } if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { return err @@ -244,9 +210,11 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} routeInfo = RouteInfo{ - Dst: dstIP, - Gw: virtualGwIP, + Dst: dstIP, + Gw: virtualGwIP, + DevName: linkToName, } + if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { return err } @@ -255,7 +223,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { // Helper that creates arp entry for the current NS which maps the virtual // gateway to destMac -func (client *NativeEndpointClient) AddDefaultArp(name string, destMac string) error { +func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", @@ -265,7 +233,7 @@ func (client *NativeEndpointClient) AddDefaultArp(name string, destMac string) e return err } if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, - name, //What is the purpose of name? + interfaceName, //What is the purpose of name? virtualGwNet.IP, hardwareAddr, false); err != nil { @@ -293,8 +261,17 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf return nil } func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + log.Printf("Setting NS to container path %d", uintptr(client.vnetNS)) + contNS, err := netns.GetFromPath(epInfo.NetNsPath) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + err = netns.Set(contNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } log.Printf("Assign IPs to container veth interface") - if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + if err = client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -306,10 +283,10 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } log.Printf("Container NS add arp entry") - if err := client.AddDefaultArp(epInfo.NetNsPath, client.vnetMac.String()); err != nil { + if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { return newErrorNativeEndpointClient(err.Error()) } - + /* Ignore the resolv.conf for now log.Printf("Create resolv.conf for DNS") folder := fmt.Sprintf("/etc/netns/%s", client.vnetNSName) resolv := fmt.Sprintf("%s/resolv.conf", folder) @@ -318,7 +295,68 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo } log.Printf("Writing to resolv.conf file %s , %s", epInfo.DNS.Servers, epInfo.DNS.Servers[0]) data := []byte(fmt.Sprintf("nameserver %s", epInfo.DNS.Servers[0])) - err := os.WriteFile(resolv, data, 0644) + if err := os.WriteFile(resolv, data, 0644); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + */ + // ip route add dev + //Each endpoint only has two routes, which are the default. + + //What is arp proxy? Set up arp rules here? + //AddEndpointRules vs. ConfigureContainerInterfacesAndRoutes + currNS, err := netns.Get() + log.Printf("Current NS before switch: %v.", currNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + log.Printf("Opening vnetns %v.", client.vnetNSName) + log.Printf("Setting NS to vnet %d", uintptr(client.vnetNS)) + netns.Set(client.vnetNS) //Catch error later + currNS, err = netns.Get() + + log.Printf("Current NS after switch: %v.", currNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + var routeInfoList []RouteInfo + + // ip route add dev + // This route is needed for incoming packets to pod to route via hostveth + for _, ipAddr := range epInfo.IPAddresses { + var ( + routeInfo RouteInfo + ipNet net.IPNet + ) + + if ipAddr.IP.To4() != nil { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} + } else { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} + } + log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) + routeInfo.Dst = ipNet + routeInfoList = append(routeInfoList, routeInfo) + + } + log.Printf("Client data: ethX: %s, vnet: %s", client.ethXVethName, client.vnetVethName) + + log.Printf("Vnet NS add default/gateway routes (Assuming indempotent)") + if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + log.Printf("Vnet NS add default ARP entry (Assuming indempotent)") + if err = client.AddDefaultArp(client.ethXVethName, azureMac); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + log.Printf("Adding routes") + if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + log.Printf("Return to container NS") + err = netns.Set(contNS) if err != nil { return newErrorNativeEndpointClient(err.Error()) } From cb4574938e952324d06dd3cc87da6eb06de68230 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 7 Jul 2022 17:04:43 -0700 Subject: [PATCH 05/52] Namespace "path" is /var/run/netns/ --- network/native_endpointclient_linux.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 440d901034..1c451986e2 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -310,10 +310,26 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } - log.Printf("Opening vnetns %v.", client.vnetNSName) - log.Printf("Setting NS to vnet %d", uintptr(client.vnetNS)) - netns.Set(client.vnetNS) //Catch error later - currNS, err = netns.Get() + // Open the vnet network namespace + log.Printf("Opening vnetns %v.", fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) + ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) + if err != nil { + return err + } + defer ns.Close() + // Enter the vnet network namespace + log.Printf("Entering vnetns %v.", ns) + if err := ns.Enter(); err != nil { + return err + } + + // Exit vnet network namespace + defer func() { + log.Printf("Exiting vnetns %v.", ns) + if err := ns.Exit(); err != nil { + log.Printf("Could not exit vnetns, err:%v.", err) + } + }() log.Printf("Current NS after switch: %v.", currNS) if err != nil { From 079ebca40af19017d42ce3aebff6032e21502380 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 8 Jul 2022 14:39:55 -0700 Subject: [PATCH 06/52] Loopback set up, Remove auto kernel subnet route --- network/native_endpointclient_linux.go | 83 +++++++++++++++----------- 1 file changed, 49 insertions(+), 34 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 1c451986e2..1f6a666786 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -16,7 +16,8 @@ import ( ) const ( - azureMac = "12:34:56:78:9a:bc" + azureMac = "12:34:56:78:9a:bc" + loopbackIf = "lo" ) var errorNativeEndpointClient = errors.New("NativeEndpointClient Error") @@ -31,7 +32,6 @@ type NativeEndpointClient struct { vnetVethName string //Peer is containerVethName containerVethName string //Peer is vnetVethName - //hostPrimaryMac net.HardwareAddr vnetMac net.HardwareAddr containerMac net.HardwareAddr ethXMac net.HardwareAddr @@ -60,21 +60,20 @@ func NewNativeEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, ) *NativeEndpointClient { - log.Printf("Create new native client: eth0:%s,ethX:%s,vnet:%s,cont:%s,id:%s", + log.Printf("Create new native client: eth0:%s, ethX:%s, vnet:%s, cont:%s, id:%s", eth0VethName, ethXVethName, vnetVethName, containerVethName, vlanid) client := &NativeEndpointClient{ eth0VethName: eth0VethName, ethXVethName: ethXVethName, vnetVethName: vnetVethName, containerVethName: containerVethName, - //hostPrimaryMac: extIf.MacAddress, - vnetNSName: vnetNSName, - mode: mode, - vlanID: vlanid, - netlink: nl, - netioshim: &netio.NetIO{}, - plClient: plc, - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + vnetNSName: vnetNSName, + mode: mode, + vlanID: vlanid, + netlink: nl, + netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), } return client @@ -90,43 +89,48 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { log.Printf("Save VM namespace: %s", vmNS) client.vmNS = vmNS - log.Printf("Create the vnet namespace") + log.Printf("Create the vnet namespace and switch to it") if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { return newErrorNativeEndpointClient(err.Error()) } + log.Printf("Set current namespace to VM: %s", vmNS) - netns.Set(vmNS) + err = netns.Set(vmNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } - log.Printf("Create the host vlan link after getting %s", client.eth0VethName) + log.Printf("Create the host vlan link after getting eth0: %s", client.eth0VethName) linkAttrs := vishnetlink.NewLinkAttrs() linkAttrs.Name = client.ethXVethName - //Get parent interface index. Is the index the same regardless of lib used? + //Get parent interface index. Index is consistent across libraries. eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) if err != nil { log.Printf("Failed to get interface: %s", client.eth0VethName) return newErrorNativeEndpointClient(err.Error()) } - //Is this how you set the peer? + //Set the peer linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ LinkAttrs: linkAttrs, VlanId: client.vlanID, } - err = vishnetlink.LinkAdd(link) - if err != nil { + if err = vishnetlink.LinkAdd(link); err != nil { return newErrorNativeEndpointClient(err.Error()) } log.Printf("Move vlan link to vnet NS: %d", uintptr(client.vnetNS)) if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNS)); err != nil { return newErrorNativeEndpointClient(err.Error()) } - vishnetlink.LinkSetUp(link) + if err = vishnetlink.LinkSetUp(link); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } log.Printf("Create veth pair (automatically set to UP)") if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } - log.Printf("Move vnetVethName into vnet namespace (peer will be moved in MoveEndpointsToContainerNS)") + log.Printf("Move vnetVethName into vnet namespace") if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNS)); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -160,7 +164,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { netns.Set(client.vnetNS) currNS, err := netns.Get() - log.Printf("Current NS after switch to vnet: %v.", currNS) + log.Printf("Current NS after switch to vnet: %s", currNS) if err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -178,7 +182,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return newErrorNativeEndpointClient(err.Error()) } client.vnetMac = vnetVethIf.HardwareAddr - //Set MTU? + log.Printf("Switch NS to vm") netns.Set(client.vmNS) @@ -194,8 +198,7 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error // Helper that creates routing rules for the current NS which direct packets // to the virtual gateway ip on linkToName device interface func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { - log.Printf("Add route for virtualgwip") - // ip route add 169.254.1.1/32 dev eth0 + log.Printf("Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0)") virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ Dst: *virtualGwNet, @@ -274,11 +277,20 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo if err = client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorNativeEndpointClient(err.Error()) } + log.Printf("Remove kernel subnet route automatically added by above") + for _, ipAddr := range epInfo.IPAddresses { + _, ipnet, _ := net.ParseCIDR(ipAddr.String()) + routeInfo := RouteInfo{ + Dst: *ipnet, + Scope: netlink.RT_SCOPE_LINK, + Protocol: netlink.RTPROT_KERNEL, + } + if err := deleteRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + return newErrorTransparentEndpointClient(err.Error()) + } + } - log.Printf("Container NS add route for virtualgwip") - // ip route add 169.254.1.1/32 dev eth0, but where do you specify eth0? - // addRoutes says it adds route to containerVethName, so this appears to be a route to the container? - //Currently called from container NS + log.Printf("Container NS add route for virtual gateway ip") if err := client.AddDefaultRoutes(client.containerVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -286,6 +298,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { return newErrorNativeEndpointClient(err.Error()) } + /* Ignore the resolv.conf for now log.Printf("Create resolv.conf for DNS") folder := fmt.Sprintf("/etc/netns/%s", client.vnetNSName) @@ -299,11 +312,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } */ - // ip route add dev - //Each endpoint only has two routes, which are the default. - //What is arp proxy? Set up arp rules here? - //AddEndpointRules vs. ConfigureContainerInterfacesAndRoutes currNS, err := netns.Get() log.Printf("Current NS before switch: %v.", currNS) if err != nil { @@ -336,10 +345,16 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } + log.Printf("Setting vnet loopback state to up") + err = client.netlink.SetLinkState(loopbackIf, true) + if err != nil { + log.Printf("Failed to set loopback link state to up") + return newErrorNativeEndpointClient(err.Error()) + } + var routeInfoList []RouteInfo - // ip route add dev - // This route is needed for incoming packets to pod to route via hostveth + // Add route specifying which device the pod ip(s) are on for _, ipAddr := range epInfo.IPAddresses { var ( routeInfo RouteInfo From 5e0eb74dce3cf9aac5f1199e91c4ffe88ff6305e Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 8 Jul 2022 15:00:28 -0700 Subject: [PATCH 07/52] Cannot set link to up if it's in another NS --- network/native_endpointclient_linux.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 1f6a666786..7531375bf4 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -115,6 +115,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { LinkAttrs: linkAttrs, VlanId: client.vlanID, } + log.Printf("Add link to VM NS (automatically set to UP)") if err = vishnetlink.LinkAdd(link); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -122,9 +123,6 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNS)); err != nil { return newErrorNativeEndpointClient(err.Error()) } - if err = vishnetlink.LinkSetUp(link); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } log.Printf("Create veth pair (automatically set to UP)") if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { From 3ad9b9827d1031dbde424a76016022ec6a2dc35b Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 11 Jul 2022 12:04:16 -0700 Subject: [PATCH 08/52] Multiple containers on same VNET NS --- network/native_endpointclient_linux.go | 31 +++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 7531375bf4..21ee5a8234 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net" + "strings" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" @@ -89,9 +90,14 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { log.Printf("Save VM namespace: %s", vmNS) client.vmNS = vmNS - log.Printf("Create the vnet namespace and switch to it") - if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { - return newErrorNativeEndpointClient(err.Error()) + log.Printf("Checking if NS exists...") + var existingErr error + client.vnetNS, existingErr = netns.GetFromName(client.vnetNSName) + if existingErr != nil { + log.Printf("No existing NS detected. Creating the vnet namespace and switching to it") + if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } } log.Printf("Set current namespace to VM: %s", vmNS) @@ -116,12 +122,21 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { VlanId: client.vlanID, } log.Printf("Add link to VM NS (automatically set to UP)") - if err = vishnetlink.LinkAdd(link); err != nil { - return newErrorNativeEndpointClient(err.Error()) + existingErr = vishnetlink.LinkAdd(link) + ethXCreated := true + if existingErr != nil { + if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { + return newErrorNativeEndpointClient(err.Error()) + } else { + log.Printf("eth0.X exists") + ethXCreated = false + } } - log.Printf("Move vlan link to vnet NS: %d", uintptr(client.vnetNS)) - if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNS)); err != nil { - return newErrorNativeEndpointClient(err.Error()) + if ethXCreated { + log.Printf("Move vlan link (ethX) to vnet NS: %d", uintptr(client.vnetNS)) + if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNS)); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } } log.Printf("Create veth pair (automatically set to UP)") From 29483f6527746aeb910f69a2660277218653b51b Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 11 Jul 2022 16:09:57 -0700 Subject: [PATCH 09/52] Delete Endpoint routes on Delete --- network/native_endpointclient_linux.go | 79 ++++++++++++++++++++------ 1 file changed, 62 insertions(+), 17 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 21ee5a8234..5385a74e62 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -257,8 +257,8 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac } return nil } -func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { - +func (client *NativeEndpointClient) DeleteEndpointRules(ep *EndpointInfo) { + //Never added any endpoint rules } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { log.Printf("Moving endpoint to container NS") @@ -276,6 +276,27 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf return nil } +func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []RouteInfo { + var routeInfoList []RouteInfo + // Add route specifying which device the pod ip(s) are on + for _, ipAddr := range ipAddresses { + var ( + routeInfo RouteInfo + ipNet net.IPNet + ) + + if ipAddr.IP.To4() != nil { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} + } else { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} + } + log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) + routeInfo.Dst = ipNet + routeInfoList = append(routeInfoList, routeInfo) + + } + return routeInfoList +} func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { log.Printf("Setting NS to container path %d", uintptr(client.vnetNS)) contNS, err := netns.GetFromPath(epInfo.NetNsPath) @@ -368,22 +389,8 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo var routeInfoList []RouteInfo // Add route specifying which device the pod ip(s) are on - for _, ipAddr := range epInfo.IPAddresses { - var ( - routeInfo RouteInfo - ipNet net.IPNet - ) - - if ipAddr.IP.To4() != nil { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} - } else { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} - } - log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) - routeInfo.Dst = ipNet - routeInfoList = append(routeInfoList, routeInfo) + routeInfoList = client.GetVnetRoutes(epInfo.IPAddresses) - } log.Printf("Client data: ethX: %s, vnet: %s", client.ethXVethName, client.vnetVethName) log.Printf("Vnet NS add default/gateway routes (Assuming indempotent)") @@ -408,5 +415,43 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return nil } func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { + // Open the vnet network namespace + log.Printf("Opening vnetns %v.", fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) + ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) + if err != nil { + return err + } + defer ns.Close() + // Enter the vnet network namespace + log.Printf("Entering vnetns %v.", ns) + if err := ns.Enter(); err != nil { + return err + } + + // Exit vnet network namespace + defer func() { + log.Printf("Exiting vnetns %v.", ns) + if err := ns.Exit(); err != nil { + log.Printf("Could not exit vnetns, err:%v.", err) + } + }() + log.Printf("Enter vnet NS") + err = netns.Set(client.vnetNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + log.Printf("Removing routes") + var routeInfoList []RouteInfo + routeInfoList = client.GetVnetRoutes(ep.IPAddresses) + if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + log.Printf("Leave vnet NS") + err = netns.Set(client.vmNS) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } return nil } From e776142fb04492328f247b14f11050fa396cc69d Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 11 Jul 2022 16:48:15 -0700 Subject: [PATCH 10/52] Minimizing netns usage --- network/native_endpointclient_linux.go | 76 +++----------------------- 1 file changed, 8 insertions(+), 68 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 5385a74e62..21fac158e3 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -126,7 +126,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { ethXCreated := true if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient(existingErr.Error()) } else { log.Printf("eth0.X exists") ethXCreated = false @@ -152,18 +152,9 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { defer func() { if err != nil { log.Printf("Switching NS to vnet") - netns.Set(client.vnetNS) + log.Printf("Failure detected, deleting links...") - //Delete vnet <-> container - if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { - log.Errorf("Deleting vnetVeth failed on addendpoint failure:%v", delErr) - } - //Delete eth0 <-> eth0.X - if delErr := client.netlink.DeleteLink(client.ethXVethName); delErr != nil { - log.Errorf("Deleting hostVeth failed on addendpoint failure:%v", delErr) - } - log.Printf("Switching NS to vm") - netns.Set(client.vmNS) + } }() log.Printf("Check that container veth exists.") @@ -257,7 +248,7 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac } return nil } -func (client *NativeEndpointClient) DeleteEndpointRules(ep *EndpointInfo) { +func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { //Never added any endpoint rules } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { @@ -299,16 +290,9 @@ func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []Rou } func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { log.Printf("Setting NS to container path %d", uintptr(client.vnetNS)) - contNS, err := netns.GetFromPath(epInfo.NetNsPath) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - err = netns.Set(contNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } + log.Printf("Assign IPs to container veth interface") - if err = client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorNativeEndpointClient(err.Error()) } log.Printf("Remove kernel subnet route automatically added by above") @@ -333,26 +317,6 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } - /* Ignore the resolv.conf for now - log.Printf("Create resolv.conf for DNS") - folder := fmt.Sprintf("/etc/netns/%s", client.vnetNSName) - resolv := fmt.Sprintf("%s/resolv.conf", folder) - if _, err := os.Stat(folder); os.IsNotExist(err) { - os.MkdirAll(folder, 0700) // Create your file - } - log.Printf("Writing to resolv.conf file %s , %s", epInfo.DNS.Servers, epInfo.DNS.Servers[0]) - data := []byte(fmt.Sprintf("nameserver %s", epInfo.DNS.Servers[0])) - if err := os.WriteFile(resolv, data, 0644); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - */ - - currNS, err := netns.Get() - log.Printf("Current NS before switch: %v.", currNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - // Open the vnet network namespace log.Printf("Opening vnetns %v.", fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) @@ -374,11 +338,6 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo } }() - log.Printf("Current NS after switch: %v.", currNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - log.Printf("Setting vnet loopback state to up") err = client.netlink.SetLinkState(loopbackIf, true) if err != nil { @@ -386,10 +345,8 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } - var routeInfoList []RouteInfo - // Add route specifying which device the pod ip(s) are on - routeInfoList = client.GetVnetRoutes(epInfo.IPAddresses) + routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) log.Printf("Client data: ethX: %s, vnet: %s", client.ethXVethName, client.vnetVethName) @@ -406,12 +363,6 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } - log.Printf("Return to container NS") - err = netns.Set(contNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - return nil } func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { @@ -435,23 +386,12 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { log.Printf("Could not exit vnetns, err:%v.", err) } }() - log.Printf("Enter vnet NS") - err = netns.Set(client.vnetNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } log.Printf("Removing routes") - var routeInfoList []RouteInfo - routeInfoList = client.GetVnetRoutes(ep.IPAddresses) + routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return newErrorNativeEndpointClient(err.Error()) } - log.Printf("Leave vnet NS") - err = netns.Set(client.vmNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } return nil } From 11afe22bade7f8f1bee1733402d2331c7f86ecd2 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 12 Jul 2022 10:28:53 -0700 Subject: [PATCH 11/52] Moving NS Exec Code --- network/native_endpointclient_linux.go | 56 ++++++++++---------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 21fac158e3..d1474dcd67 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -317,29 +317,12 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo return newErrorNativeEndpointClient(err.Error()) } - // Open the vnet network namespace - log.Printf("Opening vnetns %v.", fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) - ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) - if err != nil { - return err - } - defer ns.Close() - // Enter the vnet network namespace - log.Printf("Entering vnetns %v.", ns) - if err := ns.Enter(); err != nil { - return err - } - - // Exit vnet network namespace - defer func() { - log.Printf("Exiting vnetns %v.", ns) - if err := ns.Exit(); err != nil { - log.Printf("Could not exit vnetns, err:%v.", err) - } - }() - + err := ExecuteInNS(client.vnetNSName, epInfo, client.ConfigureVnetInterfacesAndRoutes) + return err +} +func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutes(epInfo *EndpointInfo) error { log.Printf("Setting vnet loopback state to up") - err = client.netlink.SetLinkState(loopbackIf, true) + err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { log.Printf("Failed to set loopback link state to up") return newErrorNativeEndpointClient(err.Error()) @@ -362,13 +345,25 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return newErrorNativeEndpointClient(err.Error()) } + return err +} +func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { + return ExecuteInNS(client.vnetNSName, ep, client.DeleteEndpointsImpl) +} +func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { + log.Printf("Removing routes") + routeInfoList := client.GetVnetRoutes(ep.IPAddresses) + if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } return nil } -func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { + +func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { // Open the vnet network namespace - log.Printf("Opening vnetns %v.", fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) - ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", client.vnetNSName)) + log.Printf("Opening ns %v.", fmt.Sprintf("/var/run/netns/%s", nsName)) + ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", nsName)) if err != nil { return err } @@ -383,15 +378,8 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { defer func() { log.Printf("Exiting vnetns %v.", ns) if err := ns.Exit(); err != nil { - log.Printf("Could not exit vnetns, err:%v.", err) + log.Printf("Could not exit ns, err:%v.", err) } }() - - log.Printf("Removing routes") - routeInfoList := client.GetVnetRoutes(ep.IPAddresses) - if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - - return nil + return f(param) } From 1f2c5322ea6f2b0012bf2fa8fa13d7d396085cda Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 12 Jul 2022 11:22:06 -0700 Subject: [PATCH 12/52] Further minimized netns.Set usage --- network/native_endpointclient_linux.go | 55 ++++++++++++++++---------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index d1474dcd67..ad6ab1674f 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -87,6 +87,15 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if err != nil { return newErrorNativeEndpointClient(err.Error()) } + //Mostly for reference + returnedTo, err := GetCurrentThreadNamespace() + if err != nil { + log.Printf("Unable to get VM namespace: %v", err) + return newErrorNativeEndpointClient(err.Error()) + } else { + log.Printf("VM Namespace: %v", returnedTo) + } + log.Printf("Save VM namespace: %s", vmNS) client.vmNS = vmNS @@ -148,15 +157,6 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return newErrorNativeEndpointClient(err.Error()) } - //If there is a failure, delete the links - defer func() { - if err != nil { - log.Printf("Switching NS to vnet") - - log.Printf("Failure detected, deleting links...") - - } - }() log.Printf("Check that container veth exists.") containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) if err != nil { @@ -164,8 +164,14 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } client.containerMac = containerIf.HardwareAddr - log.Printf("Switch NS to vnet") - netns.Set(client.vnetNS) + err = ExecuteInNS(client.vnetNSName, epInfo, client.PopulateVnet) + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + + return nil +} +func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { currNS, err := netns.Get() log.Printf("Current NS after switch to vnet: %s", currNS) @@ -186,10 +192,6 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return newErrorNativeEndpointClient(err.Error()) } client.vnetMac = vnetVethIf.HardwareAddr - - log.Printf("Switch NS to vm") - netns.Set(client.vmNS) - return nil } func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { @@ -289,7 +291,6 @@ func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []Rou return routeInfoList } func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - log.Printf("Setting NS to container path %d", uintptr(client.vnetNS)) log.Printf("Assign IPs to container veth interface") if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { @@ -361,24 +362,38 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { } func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { + // Current namespace + returnedTo, err := GetCurrentThreadNamespace() + if err != nil { + log.Printf("[ExecuteInNS] Could not get NS we are in: %v", err) + } else { + log.Printf("[ExecuteInNS] In NS before switch: %v", returnedTo) + } + // Open the vnet network namespace - log.Printf("Opening ns %v.", fmt.Sprintf("/var/run/netns/%s", nsName)) + log.Printf("[ExecuteInNS] Opening ns %v.", fmt.Sprintf("/var/run/netns/%s", nsName)) ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", nsName)) if err != nil { return err } defer ns.Close() // Enter the vnet network namespace - log.Printf("Entering vnetns %v.", ns) + log.Printf("[ExecuteInNS] Entering vnetns %v.", ns) if err := ns.Enter(); err != nil { return err } // Exit vnet network namespace defer func() { - log.Printf("Exiting vnetns %v.", ns) + log.Printf("[ExecuteInNS] Exiting vnetns %v.", ns) if err := ns.Exit(); err != nil { - log.Printf("Could not exit ns, err:%v.", err) + log.Printf("[ExecuteInNS] Could not exit ns, err:%v.", err) + } + returnedTo, err := GetCurrentThreadNamespace() + if err != nil { + log.Printf("[ExecuteInNS] Could not get NS we returned to: %v", err) + } else { + log.Printf("[ExecuteInNS] Returned to NS: %v", returnedTo) } }() return f(param) From 59b44a49709f86f53e342ffc5445e5b6bb9067b0 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 12 Jul 2022 16:00:58 -0700 Subject: [PATCH 13/52] Moved helper methods down, drafted tests --- network/native_endpointclient_linux.go | 177 ++++++++-------- network/native_endpointclient_linux_test.go | 222 ++++++++++++++++++++ 2 files changed, 315 insertions(+), 84 deletions(-) create mode 100644 network/native_endpointclient_linux_test.go diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index ad6ab1674f..38093a30e0 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -93,7 +93,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { log.Printf("Unable to get VM namespace: %v", err) return newErrorNativeEndpointClient(err.Error()) } else { - log.Printf("VM Namespace: %v", returnedTo) + log.Printf("VM Namespace: %s", returnedTo.file.Name()) } log.Printf("Save VM namespace: %s", vmNS) @@ -102,10 +102,15 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { log.Printf("Checking if NS exists...") var existingErr error client.vnetNS, existingErr = netns.GetFromName(client.vnetNSName) + //If the ns does not exist, the below code will trigger to create it if existingErr != nil { - log.Printf("No existing NS detected. Creating the vnet namespace and switching to it") - if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { - return newErrorNativeEndpointClient(err.Error()) + if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { + return newErrorNativeEndpointClient(existingErr.Error()) + } else { + log.Printf("No existing NS detected. Creating the vnet namespace and switching to it") + if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { + return newErrorNativeEndpointClient(err.Error()) + } } } @@ -137,7 +142,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { return newErrorNativeEndpointClient(existingErr.Error()) } else { - log.Printf("eth0.X exists") + log.Printf("eth0.X already exists") ethXCreated = false } } @@ -166,7 +171,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { err = ExecuteInNS(client.vnetNSName, epInfo, client.PopulateVnet) if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return err } return nil @@ -201,55 +206,6 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error return nil } -// Helper that creates routing rules for the current NS which direct packets -// to the virtual gateway ip on linkToName device interface -func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { - log.Printf("Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0)") - virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) - routeInfo := RouteInfo{ - Dst: *virtualGwNet, - Scope: netlink.RT_SCOPE_LINK, - DevName: linkToName, - } - if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { - return err - } - - log.Printf("Add default route (ip route add default via 169.254.1.1 dev eth0)") - _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) - dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} - routeInfo = RouteInfo{ - Dst: dstIP, - Gw: virtualGwIP, - DevName: linkToName, - } - - if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { - return err - } - return nil -} - -// Helper that creates arp entry for the current NS which maps the virtual -// gateway to destMac -func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { - _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) - // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac - log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", - virtualGwNet.String(), destMac) - hardwareAddr, err := net.ParseMAC(destMac) - if err != nil { - return err - } - if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, - interfaceName, //What is the purpose of name? - virtualGwNet.IP, - hardwareAddr, - false); err != nil { - return fmt.Errorf("adding arp entry failed: %w", err) - } - return nil -} func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { //Never added any endpoint rules } @@ -269,27 +225,7 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf return nil } -func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []RouteInfo { - var routeInfoList []RouteInfo - // Add route specifying which device the pod ip(s) are on - for _, ipAddr := range ipAddresses { - var ( - routeInfo RouteInfo - ipNet net.IPNet - ) - - if ipAddr.IP.To4() != nil { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} - } else { - ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} - } - log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) - routeInfo.Dst = ipNet - routeInfoList = append(routeInfoList, routeInfo) - } - return routeInfoList -} func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { log.Printf("Assign IPs to container veth interface") @@ -317,7 +253,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { return newErrorNativeEndpointClient(err.Error()) } - + //Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes err := ExecuteInNS(client.vnetNSName, epInfo, client.ConfigureVnetInterfacesAndRoutes) return err } @@ -342,13 +278,86 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutes(epInfo *End if err = client.AddDefaultArp(client.ethXVethName, azureMac); err != nil { return newErrorNativeEndpointClient(err.Error()) } - log.Printf("Adding routes") + log.Printf("Adding routes to vnet specific to this container") if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return newErrorNativeEndpointClient(err.Error()) } + // Return to ConfigureContainerInterfacesAndRoutes return err } +// Helper that gets the routes in the vnet NS for a particular list of IP addresses +func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []RouteInfo { + var routeInfoList []RouteInfo + // Add route specifying which device the pod ip(s) are on + for _, ipAddr := range ipAddresses { + var ( + routeInfo RouteInfo + ipNet net.IPNet + ) + + if ipAddr.IP.To4() != nil { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)} + } else { + ipNet = net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(ipv6FullMask, ipv6Bits)} + } + log.Printf("[net] Native client adding route for the ip %v", ipNet.String()) + routeInfo.Dst = ipNet + routeInfoList = append(routeInfoList, routeInfo) + + } + return routeInfoList +} + +// Helper that creates routing rules for the current NS which direct packets +// to the virtual gateway ip on linkToName device interface +func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { + log.Printf("Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0)") + virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) + routeInfo := RouteInfo{ + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + DevName: linkToName, + } + if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { + return err + } + + log.Printf("Add default route (ip route add default via 169.254.1.1 dev eth0)") + _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) + dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} + routeInfo = RouteInfo{ + Dst: dstIP, + Gw: virtualGwIP, + DevName: linkToName, + } + + if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { + return err + } + return nil +} + +// Helper that creates arp entry for the current NS which maps the virtual +// gateway to destMac +func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { + _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) + // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac + log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", + virtualGwNet.String(), destMac) + hardwareAddr, err := net.ParseMAC(destMac) + if err != nil { + return err + } + if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, + interfaceName, //What is the purpose of name? + virtualGwNet.IP, + hardwareAddr, + false); err != nil { + return fmt.Errorf("adding arp entry failed: %w", err) + } + return nil +} func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { return ExecuteInNS(client.vnetNSName, ep, client.DeleteEndpointsImpl) } @@ -367,25 +376,25 @@ func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { if err != nil { log.Printf("[ExecuteInNS] Could not get NS we are in: %v", err) } else { - log.Printf("[ExecuteInNS] In NS before switch: %v", returnedTo) + log.Printf("[ExecuteInNS] In NS before switch: %s", returnedTo.file.Name()) } - // Open the vnet network namespace + // Open the network namespace log.Printf("[ExecuteInNS] Opening ns %v.", fmt.Sprintf("/var/run/netns/%s", nsName)) ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", nsName)) if err != nil { return err } defer ns.Close() - // Enter the vnet network namespace - log.Printf("[ExecuteInNS] Entering vnetns %v.", ns) + // Enter the network namespace + log.Printf("[ExecuteInNS] Entering vnetns %s.", ns.file.Name()) if err := ns.Enter(); err != nil { return err } - // Exit vnet network namespace + // Exit network namespace defer func() { - log.Printf("[ExecuteInNS] Exiting vnetns %v.", ns) + log.Printf("[ExecuteInNS] Exiting vnetns %s.", ns.file.Name()) if err := ns.Exit(); err != nil { log.Printf("[ExecuteInNS] Could not exit ns, err:%v.", err) } @@ -393,7 +402,7 @@ func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { if err != nil { log.Printf("[ExecuteInNS] Could not get NS we returned to: %v", err) } else { - log.Printf("[ExecuteInNS] Returned to NS: %v", returnedTo) + log.Printf("[ExecuteInNS] Returned to NS: %s", returnedTo.file.Name()) } }() return f(param) diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go new file mode 100644 index 0000000000..22e26b3dd6 --- /dev/null +++ b/network/native_endpointclient_linux_test.go @@ -0,0 +1,222 @@ +//go:build linux +// +build linux + +package network + +import ( + "net" + "testing" + + "github.com/Azure/azure-container-networking/netio" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/platform" + "github.com/stretchr/testify/require" +) + +func TestNativeAddEndpoints(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + tests := []struct { + name string + client *NativeEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Add endpoints", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: false, + }, + { + name: "Add endpoints netlink fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + netlink.ErrorMockNetlink.Error() + " : netlink fail", + }, + { + name: "Add endpoints get interface fail for primary interface", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 1), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":eth0", + }, + { + name: "Add endpoints get interface fail for getting container veth", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 2), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + }, + { + name: "get interface fail for ethX veth", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 3), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":eth0.1", + }, + { + name: "get interface fail for vnet veth", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 4), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":A1veth0", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.AddEndpoints(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + vnetMac, _ := net.ParseMAC("ab:cd:ef:12:34:56") + + tests := []struct { + name string + client *NativeEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Configure Interface and routes good path", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + vnetMac: vnetMac, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: false, + }, + { + name: "Configure Interface and routes assign ip fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + vnetMac: vnetMac, + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: "netlink fail", + }, + // Any failure of netioshim causes problems + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.ConfigureContainerInterfacesAndRoutes(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} From 6c2f6c9a6d3b4ea8b77e305d0fbc6677c43cab56 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 12 Jul 2022 16:49:27 -0700 Subject: [PATCH 14/52] Removed DevName from Route Info, more tests --- network/native_endpointclient_linux.go | 13 +++--- network/native_endpointclient_linux_test.go | 50 +++++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 38093a30e0..d27a88a547 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -315,10 +315,11 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { log.Printf("Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0)") virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ - Dst: *virtualGwNet, - Scope: netlink.RT_SCOPE_LINK, - DevName: linkToName, + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + //DevName: linkToName, } + // Difference between interface name in addRoutes and DevName: in RouteInfo? if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { return err } @@ -327,9 +328,9 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} routeInfo = RouteInfo{ - Dst: dstIP, - Gw: virtualGwIP, - DevName: linkToName, + Dst: dstIP, + Gw: virtualGwIP, + //DevName: linkToName, } if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 22e26b3dd6..637643cbc3 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -204,6 +204,56 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErr: true, wantErrMsg: "netlink fail", }, + { + name: "Configure Interface and routes container final route (2nd default route) fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + vnetMac: vnetMac, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 3), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + }, + { + name: "Configure Interface and routes vnet final route (specific to container) fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + vnetMac: vnetMac, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 6), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", + }, // Any failure of netioshim causes problems } From 0b1da850a4e3af0fd18d19f0915f1f38efd979e8 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 13 Jul 2022 10:44:48 -0700 Subject: [PATCH 15/52] Test existing vnet ns, delete endpoint --- network/native_endpointclient_linux.go | 7 +- network/native_endpointclient_linux_test.go | 89 ++++++++++++++++++++- 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index d27a88a547..72b699e9c9 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -317,7 +317,6 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { routeInfo := RouteInfo{ Dst: *virtualGwNet, Scope: netlink.RT_SCOPE_LINK, - //DevName: linkToName, } // Difference between interface name in addRoutes and DevName: in RouteInfo? if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { @@ -330,7 +329,6 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { routeInfo = RouteInfo{ Dst: dstIP, Gw: virtualGwIP, - //DevName: linkToName, } if err := addRoutes(client.netlink, client.netioshim, linkToName, []RouteInfo{routeInfo}); err != nil { @@ -340,7 +338,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { } // Helper that creates arp entry for the current NS which maps the virtual -// gateway to destMac +// gateway to destMac on a particular interfaceName func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac @@ -351,7 +349,7 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac return err } if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, - interfaceName, //What is the purpose of name? + interfaceName, virtualGwNet.IP, hardwareAddr, false); err != nil { @@ -371,6 +369,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { return nil } +// Helper function that allows executing a function with one parameter in a VM namespace func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { // Current namespace returnedTo, err := GetCurrentThreadNamespace() diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 637643cbc3..b34f53bb11 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -41,6 +41,22 @@ func TestNativeAddEndpoints(t *testing.T) { epInfo: &EndpointInfo{}, wantErr: false, }, + { + name: "Add endpoints same vnet", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth1", + containerVethName: "B2veth1", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: false, + }, { name: "Add endpoints netlink fail", client: &NativeEndpointClient{ @@ -141,6 +157,46 @@ func TestNativeAddEndpoints(t *testing.T) { }) } } +func TestNativeDeleteEndpoints(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + tests := []struct { + name string + client *NativeEndpointClient + ep *endpoint + }{ + { + name: "Delete endpoint rules good path", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + ep: &endpoint{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + tt.client.DeleteEndpoints(tt.ep) + }) + } +} func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { nl := netlink.NewMockNetlink(false, "") @@ -179,6 +235,38 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, wantErr: false, }, + { + name: "Configure Interface and routes multiple IPs", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + vnetMac: vnetMac, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + { + IP: net.ParseIP("192.168.0.6"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + { + IP: net.ParseIP("192.168.0.8"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: false, + }, { name: "Configure Interface and routes assign ip fail", client: &NativeEndpointClient{ @@ -254,7 +342,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErr: true, wantErrMsg: "NativeEndpointClient Error : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", }, - // Any failure of netioshim causes problems } for _, tt := range tests { From 000258319449dc979206d849fff24da3a0a1f5ab Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 13 Jul 2022 12:09:31 -0700 Subject: [PATCH 16/52] NetNS interface for testing --- network/native_endpointclient_linux.go | 102 +++++++++++++++++++++---- 1 file changed, 87 insertions(+), 15 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 72b699e9c9..0bead7b917 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -21,6 +21,78 @@ const ( loopbackIf = "lo" ) +//Move somewhere else +type NetnsInterface interface { + Get() (fileDescriptor uintptr, err error) + GetFromName(name string) (fileDescriptor uintptr, err error) + Set(fileDescriptor uintptr) (err error) + NewNamed(name string) (fileDescriptor uintptr, err error) +} +type Netns struct{} + +func NewNetns() *Netns { + return &Netns{} +} +func (f *Netns) Get() (uintptr, error) { + nsHandle, err := netns.Get() + return uintptr(nsHandle), err +} +func (f *Netns) GetFromName(name string) (uintptr, error) { + nsHandle, err := netns.GetFromName(name) + return uintptr(nsHandle), err +} +func (f *Netns) Set(fileDescriptor uintptr) error { + return netns.Set(netns.NsHandle(fileDescriptor)) +} +func (f *Netns) NewNamed(name string) (uintptr, error) { + nsHandle, err := netns.NewNamed(name) + return uintptr(nsHandle), err +} + +var ErrorMockNetns = errors.New("mock netns error") + +func newErrorMockNetns(errStr string) error { + return fmt.Errorf("%w : %s", ErrorMockNetns, errStr) +} + +type MockNetns struct { + failMethod int + failMessage string +} + +func NewMockNetns(failMethod int, failMessage string) *MockNetns { + return &MockNetns{ + failMethod: failMethod, + failMessage: failMessage, + } +} +func (f *MockNetns) Get() (uintptr, error) { + if f.failMethod == 1 { + return 0, newErrorMockNetns(f.failMessage) + } + return 1, nil +} +func (f *MockNetns) GetFromName(name string) (uintptr, error) { + if f.failMethod == 2 { + return 0, newErrorMockNetns(f.failMessage) + } + return 1, nil +} +func (f *MockNetns) Set(handle uintptr) error { + if f.failMethod == 3 { + return newErrorMockNetns(f.failMessage) + } + return nil +} +func (f *MockNetns) NewNamed(name string) (uintptr, error) { + if f.failMethod == 4 { + return 0, newErrorMockNetns(f.failMessage) + } + return 1, nil +} + +//End move somewhere else + var errorNativeEndpointClient = errors.New("NativeEndpointClient Error") func newErrorNativeEndpointClient(errStr string) error { @@ -37,13 +109,12 @@ type NativeEndpointClient struct { containerMac net.HardwareAddr ethXMac net.HardwareAddr - vmNS netns.NsHandle - - vnetNSName string - vnetNS netns.NsHandle + vnetNSName string + vnetNSFileDescriptor uintptr mode string vlanID int + netnsClient NetnsInterface netlink netlink.NetlinkInterface netioshim netio.NetIOInterface plClient platform.ExecClient @@ -71,6 +142,7 @@ func NewNativeEndpointClient( vnetNSName: vnetNSName, mode: mode, vlanID: vlanid, + netnsClient: NewNetns(), netlink: nl, netioshim: &netio.NetIO{}, plClient: plc, @@ -83,7 +155,7 @@ func NewNativeEndpointClient( func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { var err error log.Printf("Get VM namespace handle") - vmNS, err := netns.Get() + vmNS, err := client.netnsClient.Get() if err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -96,26 +168,26 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { log.Printf("VM Namespace: %s", returnedTo.file.Name()) } - log.Printf("Save VM namespace: %s", vmNS) - client.vmNS = vmNS - log.Printf("Checking if NS exists...") var existingErr error - client.vnetNS, existingErr = netns.GetFromName(client.vnetNSName) + vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) //If the ns does not exist, the below code will trigger to create it if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { return newErrorNativeEndpointClient(existingErr.Error()) } else { log.Printf("No existing NS detected. Creating the vnet namespace and switching to it") - if client.vnetNS, err = netns.NewNamed(client.vnetNSName); err != nil { + vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) + if err != nil { return newErrorNativeEndpointClient(err.Error()) } + } } + client.vnetNSFileDescriptor = uintptr(vnetNS) log.Printf("Set current namespace to VM: %s", vmNS) - err = netns.Set(vmNS) + err = client.netnsClient.Set(vmNS) if err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -147,8 +219,8 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } } if ethXCreated { - log.Printf("Move vlan link (ethX) to vnet NS: %d", uintptr(client.vnetNS)) - if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNS)); err != nil { + log.Printf("Move vlan link (ethX) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) + if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { return newErrorNativeEndpointClient(err.Error()) } } @@ -158,7 +230,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return newErrorNativeEndpointClient(err.Error()) } log.Printf("Move vnetVethName into vnet namespace") - if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNS)); err != nil { + if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { return newErrorNativeEndpointClient(err.Error()) } @@ -178,7 +250,7 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { - currNS, err := netns.Get() + currNS, err := client.netnsClient.Get() log.Printf("Current NS after switch to vnet: %s", currNS) if err != nil { return newErrorNativeEndpointClient(err.Error()) From 2f2ef3eab113f408f1f354a5ec82ff7244df7e00 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 13 Jul 2022 14:57:55 -0700 Subject: [PATCH 17/52] Separated tests by namespace --- network/native_endpointclient_linux.go | 43 +++++-- network/native_endpointclient_linux_test.go | 131 ++++++++++++++++---- 2 files changed, 139 insertions(+), 35 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 0bead7b917..223653cc87 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -154,6 +154,21 @@ func NewNativeEndpointClient( func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { var err error + err = client.PopulateClient(epInfo) + if err != nil { + return err + } + + err = ExecuteInNS(client.vnetNSName, epInfo, client.PopulateVnet) + if err != nil { + return err + } + + return nil +} + +// Called from AddEndpoints, Namespace: VM +func (client *NativeEndpointClient) PopulateClient(epInfo *EndpointInfo) error { log.Printf("Get VM namespace handle") vmNS, err := client.netnsClient.Get() if err != nil { @@ -240,14 +255,10 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return newErrorNativeEndpointClient(err.Error()) } client.containerMac = containerIf.HardwareAddr - - err = ExecuteInNS(client.vnetNSName, epInfo, client.PopulateVnet) - if err != nil { - return err - } - return nil } + +// Called from AddEndpoints, Namespace: Vnet func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { currNS, err := client.netnsClient.Get() @@ -299,7 +310,18 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf } func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + err := client.ConfigureContainerInterfacesAndRoutesImpl(epInfo) + if err != nil { + return err + } + //Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes + err = ExecuteInNS(client.vnetNSName, epInfo, client.ConfigureVnetInterfacesAndRoutesImpl) + return err +} + +// Called from ConfigureContainerInterfacesAndRoutes, Namespace: Container +func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { log.Printf("Assign IPs to container veth interface") if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return newErrorNativeEndpointClient(err.Error()) @@ -325,11 +347,11 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { return newErrorNativeEndpointClient(err.Error()) } - //Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes - err := ExecuteInNS(client.vnetNSName, epInfo, client.ConfigureVnetInterfacesAndRoutes) - return err + return nil } -func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutes(epInfo *EndpointInfo) error { + +// Called from ConfigureContainerInterfacesAndRoutes, Namespace: Vnet +func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { log.Printf("Setting vnet loopback state to up") err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { @@ -438,6 +460,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return newErrorNativeEndpointClient(err.Error()) } + return nil } diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index b34f53bb11..7984c5cfcd 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -33,22 +33,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), - }, - epInfo: &EndpointInfo{}, - wantErr: false, - }, - { - name: "Add endpoints same vnet", - client: &NativeEndpointClient{ - eth0VethName: "eth0", - ethXVethName: "eth0.1", - vnetVethName: "A1veth1", - containerVethName: "B2veth1", - vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -65,6 +50,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -82,6 +68,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -99,6 +86,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -108,35 +96,76 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr: true, wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.PopulateClient(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } + + tests = []struct { + name string + client *NativeEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ { - name: "get interface fail for ethX veth", + name: "Add endpoints second half", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 3), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: false, + }, + { + name: "Add endpoints netlink fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, wantErr: true, wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":eth0.1", }, { - name: "get interface fail for vnet veth", + name: "Add endpoints netlink fail", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 4), + netioshim: netio.NewMockNetIO(true, 2), }, epInfo: &EndpointInfo{}, wantErr: true, @@ -147,7 +176,7 @@ func TestNativeAddEndpoints(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.AddEndpoints(tt.epInfo) + err := tt.client.PopulateVnet(tt.epInfo) if tt.wantErr { require.Error(t, err) require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) @@ -174,6 +203,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -193,7 +223,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - tt.client.DeleteEndpoints(tt.ep) + tt.client.DeleteEndpointsImpl(tt.ep) }) } } @@ -220,6 +250,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -244,6 +275,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -276,6 +308,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -301,6 +334,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -317,8 +351,54 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErr: true, wantErrMsg: "NativeEndpointClient Error : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.ConfigureContainerInterfacesAndRoutesImpl(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } + tests = []struct { + name string + client *NativeEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Configure Interface and routes good path second half", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + vnetMac: vnetMac, + netnsClient: NewMockNetns(0, ""), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: false, + }, { - name: "Configure Interface and routes vnet final route (specific to container) fail", + name: "Configure Interface and routes final routes for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -326,10 +406,11 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, + netnsClient: NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 6), + netioshim: netio.NewMockNetIO(true, 3), }, epInfo: &EndpointInfo{ IPAddresses: []net.IPNet{ @@ -347,7 +428,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.ConfigureContainerInterfacesAndRoutes(tt.epInfo) + err := tt.client.ConfigureVnetInterfacesAndRoutesImpl(tt.epInfo) if tt.wantErr { require.Error(t, err) require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) From 1c916ebadc0c9bcc80e342780efb95d11dc1bf4a Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 14 Jul 2022 16:08:24 -0700 Subject: [PATCH 18/52] Endpoints delete if they cannot be moved into NS --- network/native_endpointclient_linux.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 223653cc87..5478cb7cb6 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -152,6 +152,7 @@ func NewNativeEndpointClient( return client } +// Adds interfaces to the vnet (created if not existing) and vm namespace func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { var err error err = client.PopulateClient(epInfo) @@ -223,6 +224,7 @@ func (client *NativeEndpointClient) PopulateClient(epInfo *EndpointInfo) error { VlanId: client.vlanID, } log.Printf("Add link to VM NS (automatically set to UP)") + // Attempting to create ethX existingErr = vishnetlink.LinkAdd(link) ethXCreated := true if existingErr != nil { @@ -236,6 +238,10 @@ func (client *NativeEndpointClient) PopulateClient(epInfo *EndpointInfo) error { if ethXCreated { log.Printf("Move vlan link (ethX) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { + log.Printf("Deleting ethX veth in VM NS due to addendpoint failure") + if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { + log.Errorf("Deleting ethX veth failed on addendpoint failure:%v", delErr) + } return newErrorNativeEndpointClient(err.Error()) } } @@ -244,8 +250,13 @@ func (client *NativeEndpointClient) PopulateClient(epInfo *EndpointInfo) error { if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { return newErrorNativeEndpointClient(err.Error()) } + log.Printf("Move vnetVethName into vnet namespace") if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { + log.Printf("Deleting vnet veth due to addendpoint failure") + if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { + log.Errorf("Deleting vnet veth failed on addendpoint failure:%v", delErr) + } return newErrorNativeEndpointClient(err.Error()) } @@ -309,6 +320,7 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf return nil } +// Adds routes, arp entries, etc. to the vnet and container namespaces func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { err := client.ConfigureContainerInterfacesAndRoutesImpl(epInfo) if err != nil { @@ -381,6 +393,7 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo } // Helper that gets the routes in the vnet NS for a particular list of IP addresses +// Example: 192.168.0.4 dev proto static func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []RouteInfo { var routeInfoList []RouteInfo // Add route specifying which device the pod ip(s) are on @@ -405,6 +418,8 @@ func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []Rou // Helper that creates routing rules for the current NS which direct packets // to the virtual gateway ip on linkToName device interface +// Route 1: 169.254.1.1 dev +// Route 2: default via 169.254.1.1 dev func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { log.Printf("Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0)") virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) @@ -433,6 +448,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { // Helper that creates arp entry for the current NS which maps the virtual // gateway to destMac on a particular interfaceName +// Example: (169.254.1.1) at 12:34:56:78:9a:bc [ether] PERM on func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac @@ -465,6 +481,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { } // Helper function that allows executing a function with one parameter in a VM namespace +// Does not work for process namespaces func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { // Current namespace returnedTo, err := GetCurrentThreadNamespace() From 580929345146b0018b2a7a4c30c5784709dd6cbf Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 14 Jul 2022 16:09:00 -0700 Subject: [PATCH 19/52] Namespace netns tests --- network/native_endpointclient_linux_test.go | 72 +++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 7984c5cfcd..459632e347 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -96,6 +96,78 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr: true, wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, + { + name: "Add endpoints get interface fail for getting container veth", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(0, ""), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 2), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + }, + { + name: "Add endpoints NetNS Get fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(1, "netns failure"), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + }, + { + name: "Add endpoints NetNS GetFromName fail (with error other than file exists)", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(2, "netns failure"), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + }, + { + name: "Add endpoints NetNS Set fail", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(3, "netns failure"), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + }, } for _, tt := range tests { From 95bbea722462c29cd3847f639713fe0aca0a6058 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 14 Jul 2022 16:09:33 -0700 Subject: [PATCH 20/52] Added Native Client to deleteEndpointImpl --- network/endpoint_linux.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index c59ee0a813..82d0e3195a 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -94,6 +94,7 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p if nw.Mode == opModeNative { log.Printf("Mode %s", nw.Mode) log.Printf("Native client") + log.Printf("NSPath: %s, DNS: %s", epInfo.NetNsPath, epInfo.DNS) ethXIfName = fmt.Sprintf("eth0.%d", vlanid) vnetNSName = fmt.Sprintf("az_ns_%d", vlanid) //hostIfName may be a misnomer as this end is in the vnet NS @@ -259,7 +260,27 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. // entering the container netns and hence works both for CNI and CNM. if ep.VlanID != 0 { epInfo := ep.getInfo() - epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) + if nw.Mode == opModeNative { + + log.Printf("Mode %s", nw.Mode) + log.Printf("Native client") + log.Printf("NSPath: %s, DNS: %s", epInfo.NetNsPath, epInfo.DNS) + ethXIfName := fmt.Sprintf("eth0.%d", ep.VlanID) + vnetNSName := fmt.Sprintf("az_ns_%d", ep.VlanID) + //hostIfName may be a misnomer as this end is in the vnet NS + epClient = NewNativeEndpointClient( + nw.extIf.Name, + ethXIfName, + ep.HostIfName, + "", + vnetNSName, + nw.Mode, + ep.VlanID, + nl, + plc) + } else { + epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) + } } else if nw.Mode != opModeTransparent { epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl, plc) } else { From 0d53e94a727971ffd94855d2ab2051d9d5400b7a Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 15 Jul 2022 09:34:07 -0700 Subject: [PATCH 21/52] Deletion of Endpoints Impl and Tests --- network/native_endpointclient_linux.go | 26 ++++++++++++ network/native_endpointclient_linux_test.go | 44 ++++++++++++++++++--- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 5478cb7cb6..02431e850e 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -27,6 +27,7 @@ type NetnsInterface interface { GetFromName(name string) (fileDescriptor uintptr, err error) Set(fileDescriptor uintptr) (err error) NewNamed(name string) (fileDescriptor uintptr, err error) + DeleteNamed(name string) (err error) } type Netns struct{} @@ -48,6 +49,9 @@ func (f *Netns) NewNamed(name string) (uintptr, error) { nsHandle, err := netns.NewNamed(name) return uintptr(nsHandle), err } +func (f *Netns) DeleteNamed(name string) error { + return netns.DeleteNamed(name) +} var ErrorMockNetns = errors.New("mock netns error") @@ -90,6 +94,12 @@ func (f *MockNetns) NewNamed(name string) (uintptr, error) { } return 1, nil } +func (f *MockNetns) DeleteNamed(name string) error { + if f.failMethod == 5 { + return newErrorMockNetns(f.failMessage) + } + return nil +} //End move somewhere else @@ -474,9 +484,25 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { log.Printf("Removing routes") routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { + log.Errorf("Failed to remove routes") return newErrorNativeEndpointClient(err.Error()) } + routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) + + if err != nil { + return newErrorNativeEndpointClient(err.Error()) + } + log.Printf("There are %d routes remaining: %v", len(routes), routes) + if len(routes) <= 2 { + // Deletes default arp, default routes, ethX + log.Printf("Deleting namespace %s", client.vnetNSName) + delErr := client.netnsClient.DeleteNamed(client.vnetNSName) + if delErr != nil { + log.Errorf("Failed to delete namespace") + return newErrorNativeEndpointClient(delErr.Error()) + } + } return nil } diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 459632e347..8bc03499da 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -263,12 +263,14 @@ func TestNativeDeleteEndpoints(t *testing.T) { plc := platform.NewMockExecClient(false) tests := []struct { - name string - client *NativeEndpointClient - ep *endpoint + name string + client *NativeEndpointClient + ep *endpoint + wantErr bool + wantErrMsg string }{ { - name: "Delete endpoint rules good path", + name: "Delete endpoint good path", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -289,13 +291,45 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, }, }, + wantErr: false, + }, + { + name: "Delete endpoint fail to delete namespace ", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + ethXVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: NewMockNetns(5, "netns failure"), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + ep: &endpoint{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + wantErr: true, + wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - tt.client.DeleteEndpointsImpl(tt.ep) + err := tt.client.DeleteEndpointsImpl(tt.ep) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } }) } } From 69176810347890d8fa7af92dc8470ec9ed0b63ee Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 15 Jul 2022 16:12:58 -0700 Subject: [PATCH 22/52] Cleaned code (Tests ok) --- network/endpoint_linux.go | 16 +-- network/native_endpointclient_linux.go | 150 ++++++++------------ network/native_endpointclient_linux_test.go | 71 ++++----- 3 files changed, 99 insertions(+), 138 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 82d0e3195a..9d600ea0d5 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -92,9 +92,7 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p if vlanid != 0 { if nw.Mode == opModeNative { - log.Printf("Mode %s", nw.Mode) log.Printf("Native client") - log.Printf("NSPath: %s, DNS: %s", epInfo.NetNsPath, epInfo.DNS) ethXIfName = fmt.Sprintf("eth0.%d", vlanid) vnetNSName = fmt.Sprintf("az_ns_%d", vlanid) //hostIfName may be a misnomer as this end is in the vnet NS @@ -261,23 +259,11 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. if ep.VlanID != 0 { epInfo := ep.getInfo() if nw.Mode == opModeNative { - - log.Printf("Mode %s", nw.Mode) log.Printf("Native client") - log.Printf("NSPath: %s, DNS: %s", epInfo.NetNsPath, epInfo.DNS) ethXIfName := fmt.Sprintf("eth0.%d", ep.VlanID) vnetNSName := fmt.Sprintf("az_ns_%d", ep.VlanID) //hostIfName may be a misnomer as this end is in the vnet NS - epClient = NewNativeEndpointClient( - nw.extIf.Name, - ethXIfName, - ep.HostIfName, - "", - vnetNSName, - nw.Mode, - ep.VlanID, - nl, - plc) + epClient = NewNativeEndpointClient(nw.extIf.Name, ethXIfName, ep.HostIfName, "", vnetNSName, nw.Mode, ep.VlanID, nl, plc) } else { epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) } diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 02431e850e..5e5203660a 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -105,8 +105,8 @@ func (f *MockNetns) DeleteNamed(name string) error { var errorNativeEndpointClient = errors.New("NativeEndpointClient Error") -func newErrorNativeEndpointClient(errStr string) error { - return fmt.Errorf("%w : %s", errorNativeEndpointClient, errStr) +func newErrorNativeEndpointClient(msg string, errStr string) error { + return fmt.Errorf("%w : %s : %s", errorNativeEndpointClient, msg, errStr) } type NativeEndpointClient struct { @@ -142,7 +142,7 @@ func NewNativeEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, ) *NativeEndpointClient { - log.Printf("Create new native client: eth0:%s, ethX:%s, vnet:%s, cont:%s, id:%s", + log.Printf("[native] Create new native client: eth0:%s, ethX:%s, vnet:%s, cont:%s, id:%s", eth0VethName, ethXVethName, vnetVethName, containerVethName, vlanid) client := &NativeEndpointClient{ eth0VethName: eth0VethName, @@ -165,11 +165,12 @@ func NewNativeEndpointClient( // Adds interfaces to the vnet (created if not existing) and vm namespace func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { var err error - err = client.PopulateClient(epInfo) + // VM Namespace + err = client.PopulateVM(epInfo) if err != nil { return err } - + // VNET Namespace err = ExecuteInNS(client.vnetNSName, epInfo, client.PopulateVnet) if err != nil { return err @@ -179,101 +180,94 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } // Called from AddEndpoints, Namespace: VM -func (client *NativeEndpointClient) PopulateClient(epInfo *EndpointInfo) error { - log.Printf("Get VM namespace handle") +func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { vmNS, err := client.netnsClient.Get() if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to get VM NS handle", err.Error()) } - //Mostly for reference + // Inform current namespace returnedTo, err := GetCurrentThreadNamespace() if err != nil { - log.Printf("Unable to get VM namespace: %v", err) - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to get VM NS", err.Error()) } else { - log.Printf("VM Namespace: %s", returnedTo.file.Name()) + log.Printf("[native] VM Namespace: %s", returnedTo.file.Name()) } - log.Printf("Checking if NS exists...") + log.Printf("[native] Checking if NS exists...") var existingErr error vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) - //If the ns does not exist, the below code will trigger to create it + // If the ns does not exist, the below code will trigger to create it if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { - return newErrorNativeEndpointClient(existingErr.Error()) + return newErrorNativeEndpointClient("Error other than vnet NS doesn't exist", existingErr.Error()) } else { - log.Printf("No existing NS detected. Creating the vnet namespace and switching to it") + log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to create vnet NS", err.Error()) } } + } else { + log.Printf("[native] Existing NS detected.") } client.vnetNSFileDescriptor = uintptr(vnetNS) - log.Printf("Set current namespace to VM: %s", vmNS) err = client.netnsClient.Set(vmNS) if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to set current NS to VM", err.Error()) } - log.Printf("Create the host vlan link after getting eth0: %s", client.eth0VethName) + log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) linkAttrs := vishnetlink.NewLinkAttrs() linkAttrs.Name = client.ethXVethName - //Get parent interface index. Index is consistent across libraries. + // Get parent interface index. Index is consistent across libraries. eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) if err != nil { - log.Printf("Failed to get interface: %s", client.eth0VethName) - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to get eth0 interface", err.Error()) } - //Set the peer + // Set the peer linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ LinkAttrs: linkAttrs, VlanId: client.vlanID, } - log.Printf("Add link to VM NS (automatically set to UP)") + log.Printf("[native] Attempting to create eth0.X link in VM NS") // Attempting to create ethX existingErr = vishnetlink.LinkAdd(link) ethXCreated := true if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { - return newErrorNativeEndpointClient(existingErr.Error()) + return newErrorNativeEndpointClient("Error other than eth0.X already exists", existingErr.Error()) } else { - log.Printf("eth0.X already exists") + log.Printf("[native] eth0.X already exists") ethXCreated = false } } if ethXCreated { - log.Printf("Move vlan link (ethX) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) + log.Printf("[native] Move vlan link (eth0.X) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { - log.Printf("Deleting ethX veth in VM NS due to addendpoint failure") if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { log.Errorf("Deleting ethX veth failed on addendpoint failure:%v", delErr) } - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Deleting ethX veth in VM NS due to addendpoint failure", err.Error()) } } - log.Printf("Create veth pair (automatically set to UP)") if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to create veth pair", err.Error()) } - log.Printf("Move vnetVethName into vnet namespace") if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { - log.Printf("Deleting vnet veth due to addendpoint failure") if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { log.Errorf("Deleting vnet veth failed on addendpoint failure:%v", delErr) } - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to move vnetVethName into vnet NS, deleting", err.Error()) } - log.Printf("Check that container veth exists.") containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Container veth does not exist", err.Error()) } client.containerMac = containerIf.HardwareAddr return nil @@ -281,24 +275,15 @@ func (client *NativeEndpointClient) PopulateClient(epInfo *EndpointInfo) error { // Called from AddEndpoints, Namespace: Vnet func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { - - currNS, err := client.netnsClient.Get() - log.Printf("Current NS after switch to vnet: %s", currNS) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) - } - - log.Printf("Check that (eth0.X) exists") ethXVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("eth0.X doesn't exist", err.Error()) } client.ethXMac = ethXVethIf.HardwareAddr - log.Printf("Check that vnet veth exists") vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("vnet veth doesn't exist", err.Error()) } client.vnetMac = vnetVethIf.HardwareAddr return nil @@ -314,16 +299,14 @@ func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { //Never added any endpoint rules } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { - log.Printf("Moving endpoint to container NS") if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to move endpoint to container NS", err.Error()) } return nil } func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - log.Printf("Setup container interface") if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to setup container interface", err.Error()) } client.containerVethName = epInfo.IfName @@ -332,23 +315,24 @@ func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInf // Adds routes, arp entries, etc. to the vnet and container namespaces func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + // Container NS err := client.ConfigureContainerInterfacesAndRoutesImpl(epInfo) if err != nil { return err } - //Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes + // Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes err = ExecuteInNS(client.vnetNSName, epInfo, client.ConfigureVnetInterfacesAndRoutesImpl) return err } // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Container func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { - log.Printf("Assign IPs to container veth interface") + if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to assign IPs to container veth interface", err.Error()) } - log.Printf("Remove kernel subnet route automatically added by above") + // kernel subnet route auto added by above call must be removed for _, ipAddr := range epInfo.IPAddresses { _, ipnet, _ := net.ParseCIDR(ipAddr.String()) routeInfo := RouteInfo{ @@ -357,46 +341,38 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(ep Protocol: netlink.RTPROT_KERNEL, } if err := deleteRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { - return newErrorTransparentEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to remove kernel subnet route", err.Error()) } } - log.Printf("Container NS add route for virtual gateway ip") if err := client.AddDefaultRoutes(client.containerVethName); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed Container NS add default routes", err.Error()) } - log.Printf("Container NS add arp entry") if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed Container NS add default arp", err.Error()) } return nil } // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Vnet func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { - log.Printf("Setting vnet loopback state to up") + err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { - log.Printf("Failed to set loopback link state to up") - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to set loopback link state to up", err.Error()) } // Add route specifying which device the pod ip(s) are on routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) - log.Printf("Client data: ethX: %s, vnet: %s", client.ethXVethName, client.vnetVethName) - - log.Printf("Vnet NS add default/gateway routes (Assuming indempotent)") if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed vnet NS add default/gateway routes (indempotent)", err.Error()) } - log.Printf("Vnet NS add default ARP entry (Assuming indempotent)") if err = client.AddDefaultArp(client.ethXVethName, azureMac); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed vnet NS add default ARP entry (idempotent)", err.Error()) } - log.Printf("Adding routes to vnet specific to this container") if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed adding routes to vnet specific to this container", err.Error()) } // Return to ConfigureContainerInterfacesAndRoutes return err @@ -431,7 +407,7 @@ func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []Rou // Route 1: 169.254.1.1 dev // Route 2: default via 169.254.1.1 dev func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { - log.Printf("Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0)") + // Add route for virtualgwip (ip route add 169.254.1.1/32 dev eth0) virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) routeInfo := RouteInfo{ Dst: *virtualGwNet, @@ -442,7 +418,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { return err } - log.Printf("Add default route (ip route add default via 169.254.1.1 dev eth0)") + // Add default route (ip route add default via 169.254.1.1 dev eth0) _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} routeInfo = RouteInfo{ @@ -457,11 +433,10 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { } // Helper that creates arp entry for the current NS which maps the virtual -// gateway to destMac on a particular interfaceName +// gateway (169.254.1.1) to destMac on a particular interfaceName // Example: (169.254.1.1) at 12:34:56:78:9a:bc [ether] PERM on func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) - // arp -s 169.254.1.1 12:34:56:78:9a:bc - add static arp entry for virtualgwip to hostveth interface mac log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", virtualGwNet.String(), destMac) hardwareAddr, err := net.ParseMAC(destMac) @@ -481,26 +456,23 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { return ExecuteInNS(client.vnetNSName, ep, client.DeleteEndpointsImpl) } func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { - log.Printf("Removing routes") routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - log.Errorf("Failed to remove routes") - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to remove routes", err.Error()) } routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) - if err != nil { - return newErrorNativeEndpointClient(err.Error()) + return newErrorNativeEndpointClient("Failed to get route list", err.Error()) } - log.Printf("There are %d routes remaining: %v", len(routes), routes) + log.Printf("[native] There are %d routes remaining: %v", len(routes), routes) if len(routes) <= 2 { - // Deletes default arp, default routes, ethX - log.Printf("Deleting namespace %s", client.vnetNSName) + // Deletes default arp, default routes, ethX; there are two default routes + // so when we have <= 2 routes left, no containers use this namespace + log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) if delErr != nil { - log.Errorf("Failed to delete namespace") - return newErrorNativeEndpointClient(delErr.Error()) + return newErrorNativeEndpointClient("Failed to delete namespace", delErr.Error()) } } return nil @@ -512,7 +484,7 @@ func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { // Current namespace returnedTo, err := GetCurrentThreadNamespace() if err != nil { - log.Printf("[ExecuteInNS] Could not get NS we are in: %v", err) + log.Errorf("[ExecuteInNS] Could not get NS we are in: %v", err) } else { log.Printf("[ExecuteInNS] In NS before switch: %s", returnedTo.file.Name()) } @@ -534,11 +506,11 @@ func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { defer func() { log.Printf("[ExecuteInNS] Exiting vnetns %s.", ns.file.Name()) if err := ns.Exit(); err != nil { - log.Printf("[ExecuteInNS] Could not exit ns, err:%v.", err) + log.Errorf("[ExecuteInNS] Could not exit ns, err:%v.", err) } returnedTo, err := GetCurrentThreadNamespace() if err != nil { - log.Printf("[ExecuteInNS] Could not get NS we returned to: %v", err) + log.Errorf("[ExecuteInNS] Could not get NS we returned to: %v", err) } else { log.Printf("[ExecuteInNS] Returned to NS: %s", returnedTo.file.Name()) } diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 8bc03499da..0278f47fa8 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -25,15 +25,16 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr bool wantErrMsg string }{ + // Populating VM with data and creating interfaces/links { - name: "Add endpoints", + name: "Add endpoints no existing vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: NewMockNetns(2, "no such file or directory"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -43,7 +44,7 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr: false, }, { - name: "Add endpoints netlink fail", + name: "Add endpoints with existing vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -51,17 +52,16 @@ func TestNativeAddEndpoints(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", netnsClient: NewMockNetns(0, ""), - netlink: netlink.NewMockNetlink(true, "netlink fail"), + netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, - epInfo: &EndpointInfo{}, - wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + netlink.ErrorMockNetlink.Error() + " : netlink fail", + epInfo: &EndpointInfo{}, + wantErr: false, }, { - name: "Add endpoints get interface fail for primary interface", + name: "Add endpoints netlink fail", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -69,17 +69,17 @@ func TestNativeAddEndpoints(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", netnsClient: NewMockNetns(0, ""), - netlink: netlink.NewMockNetlink(false, ""), + netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 1), + netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":eth0", + wantErrMsg: "NativeEndpointClient Error : Failed to move vnetVethName into vnet NS, deleting : " + netlink.ErrorMockNetlink.Error() + " : netlink fail", }, { - name: "Add endpoints get interface fail for getting container veth", + name: "Add endpoints get interface fail for primary interface (eth0)", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -90,11 +90,11 @@ func TestNativeAddEndpoints(t *testing.T) { netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 2), + netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + wantErrMsg: "NativeEndpointClient Error : Failed to get eth0 interface : " + netio.ErrMockNetIOFail.Error() + ":eth0", }, { name: "Add endpoints get interface fail for getting container veth", @@ -112,7 +112,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + wantErrMsg: "NativeEndpointClient Error : Container veth does not exist : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, { name: "Add endpoints NetNS Get fail", @@ -130,10 +130,10 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Failed to get VM NS handle : " + ErrorMockNetns.Error() + " : netns failure", }, { - name: "Add endpoints NetNS GetFromName fail (with error other than file exists)", + name: "Add endpoints NetNS GetFromName fail (with error other than file does not exists)", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -148,7 +148,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Error other than vnet NS doesn't exist : " + ErrorMockNetns.Error() + " : netns failure", }, { name: "Add endpoints NetNS Set fail", @@ -166,14 +166,14 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Failed to set current NS to VM : " + ErrorMockNetns.Error() + " : netns failure", }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.PopulateClient(tt.epInfo) + err := tt.client.PopulateVM(tt.epInfo) if tt.wantErr { require.Error(t, err) require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) @@ -190,6 +190,7 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr bool wantErrMsg string }{ + // Populate the client with information from the vnet and set up vnet { name: "Add endpoints second half", client: &NativeEndpointClient{ @@ -208,7 +209,7 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr: false, }, { - name: "Add endpoints netlink fail", + name: "Add endpoints fail check eth0.X exists", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -223,10 +224,10 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":eth0.1", + wantErrMsg: "NativeEndpointClient Error : eth0.X doesn't exist : " + netio.ErrMockNetIOFail.Error() + ":eth0.1", }, { - name: "Add endpoints netlink fail", + name: "Add endpoints fail check vnet veth exists", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -241,7 +242,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":A1veth0", + wantErrMsg: "NativeEndpointClient Error : vnet veth doesn't exist : " + netio.ErrMockNetIOFail.Error() + ":A1veth0", }, } @@ -293,8 +294,9 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, wantErr: false, }, + // You must have <= 2 ip routes on your machine for this to pass { - name: "Delete endpoint fail to delete namespace ", + name: "Delete endpoint fail to delete namespace", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -316,7 +318,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Failed to delete namespace : " + ErrorMockNetns.Error() + " : netns failure", }, } @@ -348,7 +350,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErrMsg string }{ { - name: "Configure Interface and routes good path", + name: "Configure interface and routes good path for container", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -373,7 +375,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErr: false, }, { - name: "Configure Interface and routes multiple IPs", + name: "Configure interface and routes multiple IPs", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -406,7 +408,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErr: false, }, { - name: "Configure Interface and routes assign ip fail", + name: "Configure interface and routes assign ip fail", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -432,7 +434,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErrMsg: "netlink fail", }, { - name: "Configure Interface and routes container final route (2nd default route) fail", + name: "Configure interface and routes container 2nd default route added fail", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -455,7 +457,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + wantErrMsg: "NativeEndpointClient Error : Failed Container NS add default routes : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, } @@ -479,7 +481,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErrMsg string }{ { - name: "Configure Interface and routes good path second half", + name: "Configure interface and routes good path for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -504,7 +506,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErr: false, }, { - name: "Configure Interface and routes final routes for vnet", + // fail route that tells which device container ip is on for vnet + name: "Configure interface and routes fail final routes for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", ethXVethName: "eth0.1", @@ -527,7 +530,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", + wantErrMsg: "NativeEndpointClient Error : Failed adding routes to vnet specific to this container : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", }, } From 0e91d538f343d719a620e20028607c0167030cf8 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 18 Jul 2022 10:15:00 -0700 Subject: [PATCH 23/52] Moved mock/netns to package (Tests ok) --- netns/mocknetns.go | 55 ++++++++++++ netns/netns.go | 28 ++++++ netns/netnsinterface.go | 9 ++ network/native_endpointclient_linux.go | 97 ++------------------- network/native_endpointclient_linux_test.go | 47 +++++----- 5 files changed, 124 insertions(+), 112 deletions(-) create mode 100644 netns/mocknetns.go create mode 100644 netns/netns.go create mode 100644 netns/netnsinterface.go diff --git a/netns/mocknetns.go b/netns/mocknetns.go new file mode 100644 index 0000000000..fed1b4b2c4 --- /dev/null +++ b/netns/mocknetns.go @@ -0,0 +1,55 @@ +package netns + +import ( + "errors" + "fmt" +) + +var ErrorMockNetns = errors.New("mock netns error") + +func newErrorMockNetns(errStr string) error { + return fmt.Errorf("%w : %s", ErrorMockNetns, errStr) +} + +type MockNetns struct { + failMethod int + failMessage string +} + +func NewMockNetns(failMethod int, failMessage string) *MockNetns { + return &MockNetns{ + failMethod: failMethod, + failMessage: failMessage, + } +} + +func (f *MockNetns) Get() (uintptr, error) { + if f.failMethod == 1 { + return 0, newErrorMockNetns(f.failMessage) + } + return 1, nil +} +func (f *MockNetns) GetFromName(name string) (uintptr, error) { + if f.failMethod == 2 { + return 0, newErrorMockNetns(f.failMessage) + } + return 1, nil +} +func (f *MockNetns) Set(handle uintptr) error { + if f.failMethod == 3 { + return newErrorMockNetns(f.failMessage) + } + return nil +} +func (f *MockNetns) NewNamed(name string) (uintptr, error) { + if f.failMethod == 4 { + return 0, newErrorMockNetns(f.failMessage) + } + return 1, nil +} +func (f *MockNetns) DeleteNamed(name string) error { + if f.failMethod == 5 { + return newErrorMockNetns(f.failMessage) + } + return nil +} diff --git a/netns/netns.go b/netns/netns.go new file mode 100644 index 0000000000..61a5655bca --- /dev/null +++ b/netns/netns.go @@ -0,0 +1,28 @@ +package netns + +import "github.com/vishvananda/netns" + +type Netns struct{} + +func NewNetns() *Netns { + return &Netns{} +} + +func (f *Netns) Get() (uintptr, error) { + nsHandle, err := netns.Get() + return uintptr(nsHandle), err +} +func (f *Netns) GetFromName(name string) (uintptr, error) { + nsHandle, err := netns.GetFromName(name) + return uintptr(nsHandle), err +} +func (f *Netns) Set(fileDescriptor uintptr) error { + return netns.Set(netns.NsHandle(fileDescriptor)) +} +func (f *Netns) NewNamed(name string) (uintptr, error) { + nsHandle, err := netns.NewNamed(name) + return uintptr(nsHandle), err +} +func (f *Netns) DeleteNamed(name string) error { + return netns.DeleteNamed(name) +} diff --git a/netns/netnsinterface.go b/netns/netnsinterface.go new file mode 100644 index 0000000000..7960d5fd21 --- /dev/null +++ b/netns/netnsinterface.go @@ -0,0 +1,9 @@ +package netns + +type NetnsInterface interface { + Get() (fileDescriptor uintptr, err error) + GetFromName(name string) (fileDescriptor uintptr, err error) + Set(fileDescriptor uintptr) (err error) + NewNamed(name string) (fileDescriptor uintptr, err error) + DeleteNamed(name string) (err error) +} diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 5e5203660a..dfb7772ccd 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -9,100 +9,19 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" vishnetlink "github.com/vishvananda/netlink" - "github.com/vishvananda/netns" ) const ( - azureMac = "12:34:56:78:9a:bc" - loopbackIf = "lo" + azureMac = "12:34:56:78:9a:bc" // Packets leaving the VM should have this MAC + loopbackIf = "lo" // The name of the loopback interface + numDefaultRoutes = 2 // VNET NS, when no containers use it, has this many routes ) -//Move somewhere else -type NetnsInterface interface { - Get() (fileDescriptor uintptr, err error) - GetFromName(name string) (fileDescriptor uintptr, err error) - Set(fileDescriptor uintptr) (err error) - NewNamed(name string) (fileDescriptor uintptr, err error) - DeleteNamed(name string) (err error) -} -type Netns struct{} - -func NewNetns() *Netns { - return &Netns{} -} -func (f *Netns) Get() (uintptr, error) { - nsHandle, err := netns.Get() - return uintptr(nsHandle), err -} -func (f *Netns) GetFromName(name string) (uintptr, error) { - nsHandle, err := netns.GetFromName(name) - return uintptr(nsHandle), err -} -func (f *Netns) Set(fileDescriptor uintptr) error { - return netns.Set(netns.NsHandle(fileDescriptor)) -} -func (f *Netns) NewNamed(name string) (uintptr, error) { - nsHandle, err := netns.NewNamed(name) - return uintptr(nsHandle), err -} -func (f *Netns) DeleteNamed(name string) error { - return netns.DeleteNamed(name) -} - -var ErrorMockNetns = errors.New("mock netns error") - -func newErrorMockNetns(errStr string) error { - return fmt.Errorf("%w : %s", ErrorMockNetns, errStr) -} - -type MockNetns struct { - failMethod int - failMessage string -} - -func NewMockNetns(failMethod int, failMessage string) *MockNetns { - return &MockNetns{ - failMethod: failMethod, - failMessage: failMessage, - } -} -func (f *MockNetns) Get() (uintptr, error) { - if f.failMethod == 1 { - return 0, newErrorMockNetns(f.failMessage) - } - return 1, nil -} -func (f *MockNetns) GetFromName(name string) (uintptr, error) { - if f.failMethod == 2 { - return 0, newErrorMockNetns(f.failMessage) - } - return 1, nil -} -func (f *MockNetns) Set(handle uintptr) error { - if f.failMethod == 3 { - return newErrorMockNetns(f.failMessage) - } - return nil -} -func (f *MockNetns) NewNamed(name string) (uintptr, error) { - if f.failMethod == 4 { - return 0, newErrorMockNetns(f.failMessage) - } - return 1, nil -} -func (f *MockNetns) DeleteNamed(name string) error { - if f.failMethod == 5 { - return newErrorMockNetns(f.failMessage) - } - return nil -} - -//End move somewhere else - var errorNativeEndpointClient = errors.New("NativeEndpointClient Error") func newErrorNativeEndpointClient(msg string, errStr string) error { @@ -124,7 +43,7 @@ type NativeEndpointClient struct { mode string vlanID int - netnsClient NetnsInterface + netnsClient netns.NetnsInterface netlink netlink.NetlinkInterface netioshim netio.NetIOInterface plClient platform.ExecClient @@ -152,7 +71,7 @@ func NewNativeEndpointClient( vnetNSName: vnetNSName, mode: mode, vlanID: vlanid, - netnsClient: NewNetns(), + netnsClient: netns.NewNetns(), netlink: nl, netioshim: &netio.NetIO{}, plClient: plc, @@ -466,9 +385,9 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { return newErrorNativeEndpointClient("Failed to get route list", err.Error()) } log.Printf("[native] There are %d routes remaining: %v", len(routes), routes) - if len(routes) <= 2 { + if len(routes) <= numDefaultRoutes { // Deletes default arp, default routes, ethX; there are two default routes - // so when we have <= 2 routes left, no containers use this namespace + // so when we have <= numDefaultRoutes routes left, no containers use this namespace log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) if delErr != nil { diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 0278f47fa8..99788dba90 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" "github.com/stretchr/testify/require" @@ -34,7 +35,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(2, "no such file or directory"), + netnsClient: netns.NewMockNetns(2, "no such file or directory"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -51,7 +52,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -68,7 +69,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -86,7 +87,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -104,7 +105,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -122,7 +123,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(1, "netns failure"), + netnsClient: netns.NewMockNetns(1, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -130,7 +131,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to get VM NS handle : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Failed to get VM NS handle : " + netns.ErrorMockNetns.Error() + " : netns failure", }, { name: "Add endpoints NetNS GetFromName fail (with error other than file does not exists)", @@ -140,7 +141,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(2, "netns failure"), + netnsClient: netns.NewMockNetns(2, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -148,7 +149,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Error other than vnet NS doesn't exist : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Error other than vnet NS doesn't exist : " + netns.ErrorMockNetns.Error() + " : netns failure", }, { name: "Add endpoints NetNS Set fail", @@ -158,7 +159,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(3, "netns failure"), + netnsClient: netns.NewMockNetns(3, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -166,7 +167,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to set current NS to VM : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Failed to set current NS to VM : " + netns.ErrorMockNetns.Error() + " : netns failure", }, } @@ -199,7 +200,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -216,7 +217,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -234,7 +235,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -278,7 +279,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -303,7 +304,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: NewMockNetns(5, "netns failure"), + netnsClient: netns.NewMockNetns(5, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -318,7 +319,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to delete namespace : " + ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "NativeEndpointClient Error : Failed to delete namespace : " + netns.ErrorMockNetns.Error() + " : netns failure", }, } @@ -358,7 +359,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -383,7 +384,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -416,7 +417,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -442,7 +443,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -489,7 +490,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -515,7 +516,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: NewMockNetns(0, ""), + netnsClient: netns.NewMockNetns(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), From 66ea8a38efea781cfbb34fd851e695f78ce96f5d Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 18 Jul 2022 15:51:23 -0700 Subject: [PATCH 24/52] Fixing Netns (wip) Moved netnsinterface to consumer package (network). Removed "Netns" from "NewNetns" and "NewMockNetns" as it is unambiguous. Changed uintptr to int and casted the int to uintptr when needed later. --- netns/mocknetns.go | 2 +- netns/netns.go | 16 ++++++++-------- netns/netnsinterface.go | 9 --------- network/native_endpointclient_linux.go | 8 ++++---- network/netnsinterface.go | 9 +++++++++ 5 files changed, 22 insertions(+), 22 deletions(-) delete mode 100644 netns/netnsinterface.go create mode 100644 network/netnsinterface.go diff --git a/netns/mocknetns.go b/netns/mocknetns.go index fed1b4b2c4..2eb9f197a6 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -16,7 +16,7 @@ type MockNetns struct { failMessage string } -func NewMockNetns(failMethod int, failMessage string) *MockNetns { +func NewMock(failMethod int, failMessage string) *MockNetns { return &MockNetns{ failMethod: failMethod, failMessage: failMessage, diff --git a/netns/netns.go b/netns/netns.go index 61a5655bca..1389db124b 100644 --- a/netns/netns.go +++ b/netns/netns.go @@ -4,24 +4,24 @@ import "github.com/vishvananda/netns" type Netns struct{} -func NewNetns() *Netns { +func New() *Netns { return &Netns{} } -func (f *Netns) Get() (uintptr, error) { +func (f *Netns) Get() (int, error) { nsHandle, err := netns.Get() - return uintptr(nsHandle), err + return int(nsHandle), err } -func (f *Netns) GetFromName(name string) (uintptr, error) { +func (f *Netns) GetFromName(name string) (int, error) { nsHandle, err := netns.GetFromName(name) - return uintptr(nsHandle), err + return int(nsHandle), err } -func (f *Netns) Set(fileDescriptor uintptr) error { +func (f *Netns) Set(fileDescriptor int) error { return netns.Set(netns.NsHandle(fileDescriptor)) } -func (f *Netns) NewNamed(name string) (uintptr, error) { +func (f *Netns) NewNamed(name string) (int, error) { nsHandle, err := netns.NewNamed(name) - return uintptr(nsHandle), err + return int(nsHandle), err } func (f *Netns) DeleteNamed(name string) error { return netns.DeleteNamed(name) diff --git a/netns/netnsinterface.go b/netns/netnsinterface.go deleted file mode 100644 index 7960d5fd21..0000000000 --- a/netns/netnsinterface.go +++ /dev/null @@ -1,9 +0,0 @@ -package netns - -type NetnsInterface interface { - Get() (fileDescriptor uintptr, err error) - GetFromName(name string) (fileDescriptor uintptr, err error) - Set(fileDescriptor uintptr) (err error) - NewNamed(name string) (fileDescriptor uintptr, err error) - DeleteNamed(name string) (err error) -} diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index dfb7772ccd..80ff607ddc 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -39,11 +39,11 @@ type NativeEndpointClient struct { ethXMac net.HardwareAddr vnetNSName string - vnetNSFileDescriptor uintptr + vnetNSFileDescriptor int mode string vlanID int - netnsClient netns.NetnsInterface + netnsClient NetnsInterface netlink netlink.NetlinkInterface netioshim netio.NetIOInterface plClient platform.ExecClient @@ -71,7 +71,7 @@ func NewNativeEndpointClient( vnetNSName: vnetNSName, mode: mode, vlanID: vlanid, - netnsClient: netns.NewNetns(), + netnsClient: netns.New(), netlink: nl, netioshim: &netio.NetIO{}, plClient: plc, @@ -130,7 +130,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { } else { log.Printf("[native] Existing NS detected.") } - client.vnetNSFileDescriptor = uintptr(vnetNS) + client.vnetNSFileDescriptor = vnetNS err = client.netnsClient.Set(vmNS) if err != nil { diff --git a/network/netnsinterface.go b/network/netnsinterface.go new file mode 100644 index 0000000000..8f6311c386 --- /dev/null +++ b/network/netnsinterface.go @@ -0,0 +1,9 @@ +package network + +type NetnsInterface interface { + Get() (fileDescriptor int, err error) + GetFromName(name string) (fileDescriptor int, err error) + Set(fileDescriptor int) (err error) + NewNamed(name string) (fileDescriptor int, err error) + DeleteNamed(name string) (err error) +} From 0546d2ba34f6b7260de8589a678383152698c52b Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 18 Jul 2022 16:54:38 -0700 Subject: [PATCH 25/52] Using errors.Wrap for error context (wip) --- netns/mocknetns.go | 5 +-- network/native_endpointclient_linux.go | 60 ++++++++++++-------------- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index 2eb9f197a6..08e69d65f3 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -1,14 +1,13 @@ package netns import ( - "errors" - "fmt" + "github.com/pkg/errors" ) var ErrorMockNetns = errors.New("mock netns error") func newErrorMockNetns(errStr string) error { - return fmt.Errorf("%w : %s", ErrorMockNetns, errStr) + return errors.Wrap(ErrorMockNetns, errStr) } type MockNetns struct { diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 80ff607ddc..153a117e85 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -1,7 +1,6 @@ package network import ( - "errors" "fmt" "net" "strings" @@ -12,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" + "github.com/pkg/errors" vishnetlink "github.com/vishvananda/netlink" ) @@ -22,12 +22,6 @@ const ( numDefaultRoutes = 2 // VNET NS, when no containers use it, has this many routes ) -var errorNativeEndpointClient = errors.New("NativeEndpointClient Error") - -func newErrorNativeEndpointClient(msg string, errStr string) error { - return fmt.Errorf("%w : %s : %s", errorNativeEndpointClient, msg, errStr) -} - type NativeEndpointClient struct { eth0VethName string //So like eth0 ethXVethName string //So like eth0.X @@ -102,12 +96,12 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { vmNS, err := client.netnsClient.Get() if err != nil { - return newErrorNativeEndpointClient("Failed to get VM NS handle", err.Error()) + return errors.Wrap(err, "Failed to get VM NS handle") } // Inform current namespace returnedTo, err := GetCurrentThreadNamespace() if err != nil { - return newErrorNativeEndpointClient("Failed to get VM NS", err.Error()) + return errors.Wrap(err, "Failed to get VM NS") } else { log.Printf("[native] VM Namespace: %s", returnedTo.file.Name()) } @@ -118,12 +112,12 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // If the ns does not exist, the below code will trigger to create it if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { - return newErrorNativeEndpointClient("Error other than vnet NS doesn't exist", existingErr.Error()) + return errors.Wrap(existingErr, "Error other than vnet NS doesn't exist") } else { log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) if err != nil { - return newErrorNativeEndpointClient("Failed to create vnet NS", err.Error()) + return errors.Wrap(err, "Failed to create vnet NS") } } @@ -134,7 +128,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { err = client.netnsClient.Set(vmNS) if err != nil { - return newErrorNativeEndpointClient("Failed to set current NS to VM", err.Error()) + return errors.Wrap(err, "Failed to set current NS to VM") } log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) @@ -143,7 +137,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Get parent interface index. Index is consistent across libraries. eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) if err != nil { - return newErrorNativeEndpointClient("Failed to get eth0 interface", err.Error()) + return errors.Wrap(err, "Failed to get eth0 interface") } // Set the peer linkAttrs.ParentIndex = eth0.Index @@ -157,7 +151,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { ethXCreated := true if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { - return newErrorNativeEndpointClient("Error other than eth0.X already exists", existingErr.Error()) + return errors.Wrap(err, "Error other than eth0.X already exists") } else { log.Printf("[native] eth0.X already exists") ethXCreated = false @@ -169,24 +163,24 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { log.Errorf("Deleting ethX veth failed on addendpoint failure:%v", delErr) } - return newErrorNativeEndpointClient("Deleting ethX veth in VM NS due to addendpoint failure", err.Error()) + return errors.Wrap(err, "Deleting ethX veth in VM NS due to addendpoint failure") } } if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { - return newErrorNativeEndpointClient("Failed to create veth pair", err.Error()) + return errors.Wrap(err, "Failed to create veth pair") } if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { log.Errorf("Deleting vnet veth failed on addendpoint failure:%v", delErr) } - return newErrorNativeEndpointClient("Failed to move vnetVethName into vnet NS, deleting", err.Error()) + return errors.Wrap(err, "Failed to move vnetVethName into vnet NS, deleting") } containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) if err != nil { - return newErrorNativeEndpointClient("Container veth does not exist", err.Error()) + return errors.Wrap(err, "Container veth does not exist") } client.containerMac = containerIf.HardwareAddr return nil @@ -196,13 +190,13 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { ethXVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) if err != nil { - return newErrorNativeEndpointClient("eth0.X doesn't exist", err.Error()) + return errors.Wrap(err, "eth0.X doesn't exist") } client.ethXMac = ethXVethIf.HardwareAddr vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) if err != nil { - return newErrorNativeEndpointClient("vnet veth doesn't exist", err.Error()) + return errors.Wrap(err, "vnet veth doesn't exist") } client.vnetMac = vnetVethIf.HardwareAddr return nil @@ -219,13 +213,13 @@ func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { - return newErrorNativeEndpointClient("Failed to move endpoint to container NS", err.Error()) + return errors.Wrap(err, "Failed to move endpoint to container NS") } return nil } func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { - return newErrorNativeEndpointClient("Failed to setup container interface", err.Error()) + return errors.Wrap(err, "Failed to setup container interface") } client.containerVethName = epInfo.IfName @@ -249,7 +243,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { - return newErrorNativeEndpointClient("Failed to assign IPs to container veth interface", err.Error()) + return errors.Wrap(err, "Failed to assign IPs to container veth interface") } // kernel subnet route auto added by above call must be removed for _, ipAddr := range epInfo.IPAddresses { @@ -260,15 +254,15 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(ep Protocol: netlink.RTPROT_KERNEL, } if err := deleteRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { - return newErrorNativeEndpointClient("Failed to remove kernel subnet route", err.Error()) + return errors.Wrap(err, "Failed to remove kernel subnet route") } } if err := client.AddDefaultRoutes(client.containerVethName); err != nil { - return newErrorNativeEndpointClient("Failed Container NS add default routes", err.Error()) + return errors.Wrap(err, "Failed Container NS add default routes") } if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { - return newErrorNativeEndpointClient("Failed Container NS add default arp", err.Error()) + return errors.Wrap(err, "Failed Container NS add default arp") } return nil } @@ -278,20 +272,20 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { - return newErrorNativeEndpointClient("Failed to set loopback link state to up", err.Error()) + return errors.Wrap(err, "Failed to set loopback link state to up") } // Add route specifying which device the pod ip(s) are on routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { - return newErrorNativeEndpointClient("Failed vnet NS add default/gateway routes (indempotent)", err.Error()) + return errors.Wrap(err, "Failed vnet NS add default/gateway routes (indempotent)") } if err = client.AddDefaultArp(client.ethXVethName, azureMac); err != nil { - return newErrorNativeEndpointClient("Failed vnet NS add default ARP entry (idempotent)", err.Error()) + return errors.Wrap(err, "Failed vnet NS add default ARP entry (idempotent)") } if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return newErrorNativeEndpointClient("Failed adding routes to vnet specific to this container", err.Error()) + return errors.Wrap(err, "Failed adding routes to vnet specific to this container") } // Return to ConfigureContainerInterfacesAndRoutes return err @@ -377,12 +371,12 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return newErrorNativeEndpointClient("Failed to remove routes", err.Error()) + return errors.Wrap(err, "Failed to remove routes") } routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) if err != nil { - return newErrorNativeEndpointClient("Failed to get route list", err.Error()) + return errors.Wrap(err, ("Failed to get route list")) } log.Printf("[native] There are %d routes remaining: %v", len(routes), routes) if len(routes) <= numDefaultRoutes { @@ -391,7 +385,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) if delErr != nil { - return newErrorNativeEndpointClient("Failed to delete namespace", delErr.Error()) + return errors.Wrap(err, "Failed to delete namespace") } } return nil From f805465ad69ef9bac175de0e68daadf19256d526 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 18 Jul 2022 17:00:02 -0700 Subject: [PATCH 26/52] Removed sentence case (wip) --- network/native_endpointclient_linux.go | 52 +++++++++++++------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 153a117e85..e2cbee70ef 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -96,12 +96,12 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { vmNS, err := client.netnsClient.Get() if err != nil { - return errors.Wrap(err, "Failed to get VM NS handle") + return errors.Wrap(err, "failed to get vm ns handle") } // Inform current namespace returnedTo, err := GetCurrentThreadNamespace() if err != nil { - return errors.Wrap(err, "Failed to get VM NS") + return errors.Wrap(err, "failed to get vm ns") } else { log.Printf("[native] VM Namespace: %s", returnedTo.file.Name()) } @@ -112,12 +112,12 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // If the ns does not exist, the below code will trigger to create it if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { - return errors.Wrap(existingErr, "Error other than vnet NS doesn't exist") + return errors.Wrap(existingErr, "error other than vnet ns doesn't exist") } else { log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) if err != nil { - return errors.Wrap(err, "Failed to create vnet NS") + return errors.Wrap(err, "failed to create vnet ns") } } @@ -128,7 +128,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { err = client.netnsClient.Set(vmNS) if err != nil { - return errors.Wrap(err, "Failed to set current NS to VM") + return errors.Wrap(err, "failed to set current ns to vm") } log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) @@ -137,7 +137,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Get parent interface index. Index is consistent across libraries. eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) if err != nil { - return errors.Wrap(err, "Failed to get eth0 interface") + return errors.Wrap(err, "failed to get eth0 interface") } // Set the peer linkAttrs.ParentIndex = eth0.Index @@ -151,7 +151,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { ethXCreated := true if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { - return errors.Wrap(err, "Error other than eth0.X already exists") + return errors.Wrap(err, "error other than eth0.x already exists") } else { log.Printf("[native] eth0.X already exists") ethXCreated = false @@ -161,26 +161,26 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { log.Printf("[native] Move vlan link (eth0.X) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { - log.Errorf("Deleting ethX veth failed on addendpoint failure:%v", delErr) + log.Errorf("deleting ethx veth failed on addendpoint failure:%v", delErr) } - return errors.Wrap(err, "Deleting ethX veth in VM NS due to addendpoint failure") + return errors.Wrap(err, "deleting ethx veth in vm ns due to addendpoint failure") } } if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { - return errors.Wrap(err, "Failed to create veth pair") + return errors.Wrap(err, "failed to create veth pair") } if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { log.Errorf("Deleting vnet veth failed on addendpoint failure:%v", delErr) } - return errors.Wrap(err, "Failed to move vnetVethName into vnet NS, deleting") + return errors.Wrap(err, "failed to move vnetVethName into vnet ns, deleting") } containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) if err != nil { - return errors.Wrap(err, "Container veth does not exist") + return errors.Wrap(err, "container veth does not exist") } client.containerMac = containerIf.HardwareAddr return nil @@ -190,7 +190,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { ethXVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) if err != nil { - return errors.Wrap(err, "eth0.X doesn't exist") + return errors.Wrap(err, "eth0.x doesn't exist") } client.ethXMac = ethXVethIf.HardwareAddr @@ -213,13 +213,13 @@ func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { - return errors.Wrap(err, "Failed to move endpoint to container NS") + return errors.Wrap(err, "failed to move endpoint to container ns") } return nil } func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { - return errors.Wrap(err, "Failed to setup container interface") + return errors.Wrap(err, "failed to setup container interface") } client.containerVethName = epInfo.IfName @@ -243,7 +243,7 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { - return errors.Wrap(err, "Failed to assign IPs to container veth interface") + return errors.Wrap(err, "failed to assign ips to container veth interface") } // kernel subnet route auto added by above call must be removed for _, ipAddr := range epInfo.IPAddresses { @@ -254,15 +254,15 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(ep Protocol: netlink.RTPROT_KERNEL, } if err := deleteRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { - return errors.Wrap(err, "Failed to remove kernel subnet route") + return errors.Wrap(err, "failed to remove kernel subnet route") } } if err := client.AddDefaultRoutes(client.containerVethName); err != nil { - return errors.Wrap(err, "Failed Container NS add default routes") + return errors.Wrap(err, "failed Container ns add default routes") } if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { - return errors.Wrap(err, "Failed Container NS add default arp") + return errors.Wrap(err, "failed Container ns add default arp") } return nil } @@ -272,20 +272,20 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { - return errors.Wrap(err, "Failed to set loopback link state to up") + return errors.Wrap(err, "failed to set loopback link state to up") } // Add route specifying which device the pod ip(s) are on routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { - return errors.Wrap(err, "Failed vnet NS add default/gateway routes (indempotent)") + return errors.Wrap(err, "failed vnet ns add default/gateway routes (indempotent)") } if err = client.AddDefaultArp(client.ethXVethName, azureMac); err != nil { - return errors.Wrap(err, "Failed vnet NS add default ARP entry (idempotent)") + return errors.Wrap(err, "failed vnet ns add default arp entry (idempotent)") } if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return errors.Wrap(err, "Failed adding routes to vnet specific to this container") + return errors.Wrap(err, "failed adding routes to vnet specific to this container") } // Return to ConfigureContainerInterfacesAndRoutes return err @@ -371,12 +371,12 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { - return errors.Wrap(err, "Failed to remove routes") + return errors.Wrap(err, "failed to remove routes") } routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) if err != nil { - return errors.Wrap(err, ("Failed to get route list")) + return errors.Wrap(err, ("failed to get route list")) } log.Printf("[native] There are %d routes remaining: %v", len(routes), routes) if len(routes) <= numDefaultRoutes { @@ -385,7 +385,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) if delErr != nil { - return errors.Wrap(err, "Failed to delete namespace") + return errors.Wrap(err, "failed to delete namespace") } } return nil From 8ef521df3c3c7d26226d059f440fca39d279a791 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:11:51 -0700 Subject: [PATCH 27/52] Removing variable predeclaration --- network/native_endpointclient_linux.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index e2cbee70ef..70766a40c5 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -77,9 +77,8 @@ func NewNativeEndpointClient( // Adds interfaces to the vnet (created if not existing) and vm namespace func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { - var err error // VM Namespace - err = client.PopulateVM(epInfo) + err := client.PopulateVM(epInfo) if err != nil { return err } @@ -107,9 +106,9 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { } log.Printf("[native] Checking if NS exists...") - var existingErr error vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) // If the ns does not exist, the below code will trigger to create it + if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { return errors.Wrap(existingErr, "error other than vnet ns doesn't exist") From e8b45af7c4421f6a1aebdc650bd6a6f1d35d5c4e Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:21:12 -0700 Subject: [PATCH 28/52] Removed NewNativeEndpointClient Directly instantiating struct because nothing special happens in NewNativeEndpointClient --- network/endpoint_linux.go | 44 ++++++++++++++++++-------- network/native_endpointclient_linux.go | 32 ------------------- 2 files changed, 31 insertions(+), 45 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 9d600ea0d5..d035c4990c 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/ovsctl" "github.com/Azure/azure-container-networking/platform" @@ -95,17 +96,21 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p log.Printf("Native client") ethXIfName = fmt.Sprintf("eth0.%d", vlanid) vnetNSName = fmt.Sprintf("az_ns_%d", vlanid) - //hostIfName may be a misnomer as this end is in the vnet NS - epClient = NewNativeEndpointClient( - nw.extIf.Name, - ethXIfName, - hostIfName, - contIfName, - vnetNSName, - nw.Mode, - vlanid, - nl, - plc) + + epClient = &NativeEndpointClient{ + eth0VethName: nw.extIf.Name, + ethXVethName: ethXIfName, + vnetVethName: hostIfName, + containerVethName: contIfName, + vnetNSName: vnetNSName, + mode: nw.Mode, + vlanID: vlanid, + netnsClient: netns.New(), + netlink: nl, + netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + } } else { log.Printf("OVS client") if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { @@ -262,8 +267,21 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. log.Printf("Native client") ethXIfName := fmt.Sprintf("eth0.%d", ep.VlanID) vnetNSName := fmt.Sprintf("az_ns_%d", ep.VlanID) - //hostIfName may be a misnomer as this end is in the vnet NS - epClient = NewNativeEndpointClient(nw.extIf.Name, ethXIfName, ep.HostIfName, "", vnetNSName, nw.Mode, ep.VlanID, nl, plc) + + epClient = &NativeEndpointClient{ + eth0VethName: nw.extIf.Name, + ethXVethName: ethXIfName, + vnetVethName: ep.HostIfName, + containerVethName: "", + vnetNSName: vnetNSName, + mode: nw.Mode, + vlanID: ep.VlanID, + netnsClient: netns.New(), + netlink: nl, + netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + } } else { epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) } diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 70766a40c5..dd4b479d37 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -8,7 +8,6 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" "github.com/pkg/errors" @@ -44,37 +43,6 @@ type NativeEndpointClient struct { netUtilsClient networkutils.NetworkUtils } -func NewNativeEndpointClient( - eth0VethName string, - ethXVethName string, - vnetVethName string, - containerVethName string, - vnetNSName string, - mode string, - vlanid int, - nl netlink.NetlinkInterface, - plc platform.ExecClient, -) *NativeEndpointClient { - log.Printf("[native] Create new native client: eth0:%s, ethX:%s, vnet:%s, cont:%s, id:%s", - eth0VethName, ethXVethName, vnetVethName, containerVethName, vlanid) - client := &NativeEndpointClient{ - eth0VethName: eth0VethName, - ethXVethName: ethXVethName, - vnetVethName: vnetVethName, - containerVethName: containerVethName, - vnetNSName: vnetNSName, - mode: mode, - vlanID: vlanid, - netnsClient: netns.New(), - netlink: nl, - netioshim: &netio.NetIO{}, - plClient: plc, - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - } - - return client -} - // Adds interfaces to the vnet (created if not existing) and vm namespace func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { // VM Namespace From cc8bf9a1fcf516446b4cb6eca20e846a52367b8d Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:30:35 -0700 Subject: [PATCH 29/52] Removed generics from ExecuteInNS --- network/native_endpointclient_linux.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index dd4b479d37..7b0cd6d4e7 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -51,12 +51,9 @@ func (client *NativeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { return err } // VNET Namespace - err = ExecuteInNS(client.vnetNSName, epInfo, client.PopulateVnet) - if err != nil { - return err - } - - return nil + return ExecuteInNS(client.vnetNSName, func() error { + return client.PopulateVnet(epInfo) + }) } // Called from AddEndpoints, Namespace: VM @@ -202,8 +199,9 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo } // Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes - err = ExecuteInNS(client.vnetNSName, epInfo, client.ConfigureVnetInterfacesAndRoutesImpl) - return err + return ExecuteInNS(client.vnetNSName, func() error { + return client.ConfigureVnetInterfacesAndRoutesImpl(epInfo) + }) } // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Container @@ -333,7 +331,9 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac return nil } func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { - return ExecuteInNS(client.vnetNSName, ep, client.DeleteEndpointsImpl) + return ExecuteInNS(client.vnetNSName, func() error { + return client.DeleteEndpointsImpl(ep) + }) } func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) @@ -360,7 +360,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { // Helper function that allows executing a function with one parameter in a VM namespace // Does not work for process namespaces -func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { +func ExecuteInNS(nsName string, f func() error) error { // Current namespace returnedTo, err := GetCurrentThreadNamespace() if err != nil { @@ -395,5 +395,5 @@ func ExecuteInNS[T any](nsName string, param *T, f func(param *T) error) error { log.Printf("[ExecuteInNS] Returned to NS: %s", returnedTo.file.Name()) } }() - return f(param) + return f() } From 71f45662de7b392a88f44b7c6094f4ae7698065d Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:43:25 -0700 Subject: [PATCH 30/52] Removed uintptr from mocknetns, tests compile Forgot to remove uintptr from mocknetns --- netns/mocknetns.go | 8 ++--- network/native_endpointclient_linux_test.go | 38 ++++++++++----------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index 08e69d65f3..4026de8ab0 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -22,25 +22,25 @@ func NewMock(failMethod int, failMessage string) *MockNetns { } } -func (f *MockNetns) Get() (uintptr, error) { +func (f *MockNetns) Get() (int, error) { if f.failMethod == 1 { return 0, newErrorMockNetns(f.failMessage) } return 1, nil } -func (f *MockNetns) GetFromName(name string) (uintptr, error) { +func (f *MockNetns) GetFromName(name string) (int, error) { if f.failMethod == 2 { return 0, newErrorMockNetns(f.failMessage) } return 1, nil } -func (f *MockNetns) Set(handle uintptr) error { +func (f *MockNetns) Set(handle int) error { if f.failMethod == 3 { return newErrorMockNetns(f.failMessage) } return nil } -func (f *MockNetns) NewNamed(name string) (uintptr, error) { +func (f *MockNetns) NewNamed(name string) (int, error) { if f.failMethod == 4 { return 0, newErrorMockNetns(f.failMessage) } diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 99788dba90..3a2f9d71b1 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -35,7 +35,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(2, "no such file or directory"), + netnsClient: netns.NewMock(2, "no such file or directory"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -52,7 +52,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -69,7 +69,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -87,7 +87,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -105,7 +105,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -123,7 +123,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(1, "netns failure"), + netnsClient: netns.NewMock(1, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -141,7 +141,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(2, "netns failure"), + netnsClient: netns.NewMock(2, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -159,7 +159,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(3, "netns failure"), + netnsClient: netns.NewMock(3, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -200,7 +200,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -217,7 +217,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -235,7 +235,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -279,7 +279,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -304,7 +304,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMockNetns(5, "netns failure"), + netnsClient: netns.NewMock(5, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -359,7 +359,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -384,7 +384,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -417,7 +417,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -443,7 +443,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -490,7 +490,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -516,7 +516,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMockNetns(0, ""), + netnsClient: netns.NewMock(0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), From 3fb5df0a79ca7ca4ad1b5ea204735f6c1d41502c Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 10:23:19 -0700 Subject: [PATCH 31/52] Fix tests, lint --- netns/mocknetns.go | 16 ++++----- network/native_endpointclient_linux.go | 40 ++++++++++----------- network/native_endpointclient_linux_test.go | 28 +++++++-------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index 4026de8ab0..940d2b1cf3 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -4,10 +4,10 @@ import ( "github.com/pkg/errors" ) -var ErrorMockNetns = errors.New("mock netns error") +var ErrorMock = errors.New("mock netns error") -func newErrorMockNetns(errStr string) error { - return errors.Wrap(ErrorMockNetns, errStr) +func newErrorMock(errStr string) error { + return errors.Wrap(ErrorMock, errStr) } type MockNetns struct { @@ -24,31 +24,31 @@ func NewMock(failMethod int, failMessage string) *MockNetns { func (f *MockNetns) Get() (int, error) { if f.failMethod == 1 { - return 0, newErrorMockNetns(f.failMessage) + return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) GetFromName(name string) (int, error) { if f.failMethod == 2 { - return 0, newErrorMockNetns(f.failMessage) + return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) Set(handle int) error { if f.failMethod == 3 { - return newErrorMockNetns(f.failMessage) + return newErrorMock(f.failMessage) } return nil } func (f *MockNetns) NewNamed(name string) (int, error) { if f.failMethod == 4 { - return 0, newErrorMockNetns(f.failMessage) + return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) DeleteNamed(name string) error { if f.failMethod == 5 { - return newErrorMockNetns(f.failMessage) + return newErrorMock(f.failMessage) } return nil } diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 7b0cd6d4e7..6ebcb709bb 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -22,10 +22,10 @@ const ( ) type NativeEndpointClient struct { - eth0VethName string //So like eth0 - ethXVethName string //So like eth0.X - vnetVethName string //Peer is containerVethName - containerVethName string //Peer is vnetVethName + eth0VethName string // So like eth0 + ethXVethName string // So like eth0.X + vnetVethName string // Peer is containerVethName + containerVethName string // Peer is vnetVethName vnetMac net.HardwareAddr containerMac net.HardwareAddr @@ -66,25 +66,24 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { returnedTo, err := GetCurrentThreadNamespace() if err != nil { return errors.Wrap(err, "failed to get vm ns") - } else { - log.Printf("[native] VM Namespace: %s", returnedTo.file.Name()) } + log.Printf("[native] VM Namespace: %s", returnedTo.file.Name()) log.Printf("[native] Checking if NS exists...") vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) // If the ns does not exist, the below code will trigger to create it - if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { + // Something else went wrong return errors.Wrap(existingErr, "error other than vnet ns doesn't exist") - } else { - log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") - vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) - if err != nil { - return errors.Wrap(err, "failed to create vnet ns") - } - } + // The vnet ns does not exist, which is okay + log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") + vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) + if err != nil { + return errors.Wrap(err, "failed to create vnet ns") + } + } else { log.Printf("[native] Existing NS detected.") } @@ -115,11 +114,12 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { ethXCreated := true if existingErr != nil { if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { + // Something else went wrong return errors.Wrap(err, "error other than eth0.x already exists") - } else { - log.Printf("[native] eth0.X already exists") - ethXCreated = false } + // The interface exists already, which is okay + log.Printf("[native] eth0.X already exists") + ethXCreated = false } if ethXCreated { log.Printf("[native] Move vlan link (eth0.X) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) @@ -224,10 +224,10 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(ep } if err := client.AddDefaultRoutes(client.containerVethName); err != nil { - return errors.Wrap(err, "failed Container ns add default routes") + return errors.Wrap(err, "failed container ns add default routes") } if err := client.AddDefaultArp(client.containerVethName, client.vnetMac.String()); err != nil { - return errors.Wrap(err, "failed Container ns add default arp") + return errors.Wrap(err, "failed container ns add default arp") } return nil } @@ -352,7 +352,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) if delErr != nil { - return errors.Wrap(err, "failed to delete namespace") + return errors.Wrap(delErr, "failed to delete namespace") } } return nil diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 3a2f9d71b1..756b11257a 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -77,7 +77,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to move vnetVethName into vnet NS, deleting : " + netlink.ErrorMockNetlink.Error() + " : netlink fail", + wantErrMsg: "failed to move vnetVethName into vnet ns, deleting: " + netlink.ErrorMockNetlink.Error() + " : netlink fail", }, { name: "Add endpoints get interface fail for primary interface (eth0)", @@ -95,7 +95,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to get eth0 interface : " + netio.ErrMockNetIOFail.Error() + ":eth0", + wantErrMsg: "failed to get eth0 interface: " + netio.ErrMockNetIOFail.Error() + ":eth0", }, { name: "Add endpoints get interface fail for getting container veth", @@ -113,7 +113,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Container veth does not exist : " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + wantErrMsg: "container veth does not exist: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, { name: "Add endpoints NetNS Get fail", @@ -131,7 +131,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to get VM NS handle : " + netns.ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "failed to get vm ns handle: netns failure: " + netns.ErrorMock.Error(), }, { name: "Add endpoints NetNS GetFromName fail (with error other than file does not exists)", @@ -149,7 +149,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Error other than vnet NS doesn't exist : " + netns.ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "error other than vnet ns doesn't exist: netns failure: " + netns.ErrorMock.Error(), }, { name: "Add endpoints NetNS Set fail", @@ -167,7 +167,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to set current NS to VM : " + netns.ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "failed to set current ns to vm: netns failure: " + netns.ErrorMock.Error(), }, } @@ -177,7 +177,7 @@ func TestNativeAddEndpoints(t *testing.T) { err := tt.client.PopulateVM(tt.epInfo) if tt.wantErr { require.Error(t, err) - require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) } else { require.NoError(t, err) } @@ -225,7 +225,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : eth0.X doesn't exist : " + netio.ErrMockNetIOFail.Error() + ":eth0.1", + wantErrMsg: "eth0.x doesn't exist: " + netio.ErrMockNetIOFail.Error() + ":eth0.1", }, { name: "Add endpoints fail check vnet veth exists", @@ -243,7 +243,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : vnet veth doesn't exist : " + netio.ErrMockNetIOFail.Error() + ":A1veth0", + wantErrMsg: "vnet veth doesn't exist: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", }, } @@ -253,7 +253,7 @@ func TestNativeAddEndpoints(t *testing.T) { err := tt.client.PopulateVnet(tt.epInfo) if tt.wantErr { require.Error(t, err) - require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) } else { require.NoError(t, err) } @@ -319,7 +319,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed to delete namespace : " + netns.ErrorMockNetns.Error() + " : netns failure", + wantErrMsg: "failed to delete namespace: netns failure: " + netns.ErrorMock.Error(), }, } @@ -329,7 +329,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { err := tt.client.DeleteEndpointsImpl(tt.ep) if tt.wantErr { require.Error(t, err) - require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) } else { require.NoError(t, err) } @@ -458,7 +458,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed Container NS add default routes : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + wantErrMsg: "failed container ns add default routes: addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", }, } @@ -531,7 +531,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "NativeEndpointClient Error : Failed adding routes to vnet specific to this container : addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", + wantErrMsg: "failed adding routes to vnet specific to this container: addRoutes failed: " + netio.ErrMockNetIOFail.Error() + ":A1veth0", }, } From 6a999f7d053abf0c5d32a53a03f8447abca9e8ec Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 13:30:02 -0700 Subject: [PATCH 32/52] Fixes from linter Works on VMSS --- network/native_endpointclient_linux.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 6ebcb709bb..2041ad892b 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -73,6 +73,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) // If the ns does not exist, the below code will trigger to create it if existingErr != nil { + log.Printf("[native] Existing response is: %s", existingErr.Error()) if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { // Something else went wrong return errors.Wrap(existingErr, "error other than vnet ns doesn't exist") @@ -113,6 +114,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { existingErr = vishnetlink.LinkAdd(link) ethXCreated := true if existingErr != nil { + log.Printf("[native] Existing response is: %s", existingErr.Error()) if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { // Something else went wrong return errors.Wrap(err, "error other than eth0.x already exists") @@ -166,14 +168,14 @@ func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { return nil } func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { - //There are no rules to add here - //Described as rules on ip addresses on the container interface + // There are no rules to add here + // Described as rules on ip addresses on the container interface return nil } func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { - //Never added any endpoint rules + // Never added any endpoint rules } func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { @@ -313,7 +315,7 @@ func (client *NativeEndpointClient) AddDefaultRoutes(linkToName string) error { // Helper that creates arp entry for the current NS which maps the virtual // gateway (169.254.1.1) to destMac on a particular interfaceName // Example: (169.254.1.1) at 12:34:56:78:9a:bc [ether] PERM on -func (client *NativeEndpointClient) AddDefaultArp(interfaceName string, destMac string) error { +func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) error { _, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) log.Printf("[net] Adding static arp for IP address %v and MAC %v in namespace", virtualGwNet.String(), destMac) From 147a7c93b7d6b41e80922b97a71c8914c35e9aea Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 13:49:56 -0700 Subject: [PATCH 33/52] Replacing references to ethX with vlan veth --- network/endpoint_linux.go | 16 ++++---- network/native_endpointclient_linux.go | 42 ++++++++++----------- network/native_endpointclient_linux_test.go | 38 +++++++++---------- 3 files changed, 47 insertions(+), 49 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index d035c4990c..9dbd21bfec 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -59,8 +59,6 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p var localIP string var epClient EndpointClient var vlanid int = 0 - var ethXIfName string - var vnetNSName string if nw.Endpoints[epInfo.Id] != nil { log.Printf("[net] Endpoint alreday exists.") @@ -94,16 +92,16 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p if vlanid != 0 { if nw.Mode == opModeNative { log.Printf("Native client") - ethXIfName = fmt.Sprintf("eth0.%d", vlanid) - vnetNSName = fmt.Sprintf("az_ns_%d", vlanid) + vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, vlanid) + vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) epClient = &NativeEndpointClient{ eth0VethName: nw.extIf.Name, - ethXVethName: ethXIfName, + vlanVethName: vlanVethName, vnetVethName: hostIfName, containerVethName: contIfName, vnetNSName: vnetNSName, - mode: nw.Mode, + nw: nw, vlanID: vlanid, netnsClient: netns.New(), netlink: nl, @@ -265,16 +263,16 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. epInfo := ep.getInfo() if nw.Mode == opModeNative { log.Printf("Native client") - ethXIfName := fmt.Sprintf("eth0.%d", ep.VlanID) + vlanVethName := fmt.Sprintf("%s.%d", nw.extIf.Name, ep.VlanID) vnetNSName := fmt.Sprintf("az_ns_%d", ep.VlanID) epClient = &NativeEndpointClient{ eth0VethName: nw.extIf.Name, - ethXVethName: ethXIfName, + vlanVethName: vlanVethName, vnetVethName: ep.HostIfName, containerVethName: "", vnetNSName: vnetNSName, - mode: nw.Mode, + nw: nw, vlanID: ep.VlanID, netnsClient: netns.New(), netlink: nl, diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 2041ad892b..dec7c0c669 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -23,18 +23,18 @@ const ( type NativeEndpointClient struct { eth0VethName string // So like eth0 - ethXVethName string // So like eth0.X + vlanVethName string // So like eth0.1 vnetVethName string // Peer is containerVethName containerVethName string // Peer is vnetVethName vnetMac net.HardwareAddr containerMac net.HardwareAddr - ethXMac net.HardwareAddr + vlanMac net.HardwareAddr vnetNSName string vnetNSFileDescriptor int - mode string + nw *network vlanID int netnsClient NetnsInterface netlink netlink.NetlinkInterface @@ -97,7 +97,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) linkAttrs := vishnetlink.NewLinkAttrs() - linkAttrs.Name = client.ethXVethName + linkAttrs.Name = client.vlanVethName // Get parent interface index. Index is consistent across libraries. eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) if err != nil { @@ -109,27 +109,27 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { LinkAttrs: linkAttrs, VlanId: client.vlanID, } - log.Printf("[native] Attempting to create eth0.X link in VM NS") - // Attempting to create ethX + log.Printf("[native] Attempting to create %s link in VM NS", client.vlanVethName) + // Attempting to create vlan veth existingErr = vishnetlink.LinkAdd(link) - ethXCreated := true + vlanVethCreated := true if existingErr != nil { log.Printf("[native] Existing response is: %s", existingErr.Error()) if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { // Something else went wrong - return errors.Wrap(err, "error other than eth0.x already exists") + return errors.Wrap(err, "error other than vlan veth already exists") } // The interface exists already, which is okay - log.Printf("[native] eth0.X already exists") - ethXCreated = false + log.Printf("[native] %s already exists", client.vlanVethName) + vlanVethCreated = false } - if ethXCreated { - log.Printf("[native] Move vlan link (eth0.X) to vnet NS: %d", uintptr(client.vnetNSFileDescriptor)) - if err = client.netlink.SetLinkNetNs(client.ethXVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { + if vlanVethCreated { + log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) + if err = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { - log.Errorf("deleting ethx veth failed on addendpoint failure:%v", delErr) + log.Errorf("deleting vlan veth failed on addendpoint failure:%v", delErr) } - return errors.Wrap(err, "deleting ethx veth in vm ns due to addendpoint failure") + return errors.Wrap(err, "deleting vlan veth in vm ns due to addendpoint failure") } } @@ -154,11 +154,11 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Called from AddEndpoints, Namespace: Vnet func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { - ethXVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.ethXVethName) + vlanVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vlanVethName) if err != nil { - return errors.Wrap(err, "eth0.x doesn't exist") + return errors.Wrap(err, "vlan veth doesn't exist") } - client.ethXMac = ethXVethIf.HardwareAddr + client.vlanMac = vlanVethIf.HardwareAddr vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) if err != nil { @@ -245,10 +245,10 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo // Add route specifying which device the pod ip(s) are on routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) - if err = client.AddDefaultRoutes(client.ethXVethName); err != nil { + if err = client.AddDefaultRoutes(client.vlanVethName); err != nil { return errors.Wrap(err, "failed vnet ns add default/gateway routes (indempotent)") } - if err = client.AddDefaultArp(client.ethXVethName, azureMac); err != nil { + if err = client.AddDefaultArp(client.vlanVethName, azureMac); err != nil { return errors.Wrap(err, "failed vnet ns add default arp entry (idempotent)") } if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { @@ -349,7 +349,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { } log.Printf("[native] There are %d routes remaining: %v", len(routes), routes) if len(routes) <= numDefaultRoutes { - // Deletes default arp, default routes, ethX; there are two default routes + // Deletes default arp, default routes, vlan veth; there are two default routes // so when we have <= numDefaultRoutes routes left, no containers use this namespace log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) delErr := client.netnsClient.DeleteNamed(client.vnetNSName) diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 756b11257a..15c462d29d 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -31,7 +31,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints no existing vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -48,7 +48,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints with existing vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -65,7 +65,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints netlink fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -83,7 +83,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints get interface fail for primary interface (eth0)", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -101,7 +101,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints get interface fail for getting container veth", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -119,7 +119,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints NetNS Get fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -137,7 +137,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints NetNS GetFromName fail (with error other than file does not exists)", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -155,7 +155,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints NetNS Set fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -196,7 +196,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints second half", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -213,7 +213,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints fail check eth0.X exists", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -231,7 +231,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints fail check vnet veth exists", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -275,7 +275,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { name: "Delete endpoint good path", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -300,7 +300,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { name: "Delete endpoint fail to delete namespace", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -354,7 +354,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes good path for container", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -379,7 +379,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes multiple IPs", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -412,7 +412,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes assign ip fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -438,7 +438,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes container 2nd default route added fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -485,7 +485,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes good path for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -511,7 +511,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes fail final routes for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", - ethXVethName: "eth0.1", + vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", From 08c6ed501c32177e746adb60ee77d561e419392c Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 13:53:35 -0700 Subject: [PATCH 34/52] Removed unnecessary log --- network/native_endpointclient_linux.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index dec7c0c669..573d5fd924 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -62,12 +62,6 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { if err != nil { return errors.Wrap(err, "failed to get vm ns handle") } - // Inform current namespace - returnedTo, err := GetCurrentThreadNamespace() - if err != nil { - return errors.Wrap(err, "failed to get vm ns") - } - log.Printf("[native] VM Namespace: %s", returnedTo.file.Name()) log.Printf("[native] Checking if NS exists...") vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) From 947a6a9d38e57332309e33e1d35b23f4dab5607b Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 13:55:03 -0700 Subject: [PATCH 35/52] Removed unnecessary mac, fix tests --- network/native_endpointclient_linux.go | 5 +---- network/native_endpointclient_linux_test.go | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 573d5fd924..b5c359d3a0 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -29,7 +29,6 @@ type NativeEndpointClient struct { vnetMac net.HardwareAddr containerMac net.HardwareAddr - vlanMac net.HardwareAddr vnetNSName string vnetNSFileDescriptor int @@ -148,12 +147,10 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Called from AddEndpoints, Namespace: Vnet func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { - vlanVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vlanVethName) + _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanVethName) if err != nil { return errors.Wrap(err, "vlan veth doesn't exist") } - client.vlanMac = vlanVethIf.HardwareAddr - vnetVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) if err != nil { return errors.Wrap(err, "vnet veth doesn't exist") diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 15c462d29d..da708bae2d 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -193,7 +193,7 @@ func TestNativeAddEndpoints(t *testing.T) { }{ // Populate the client with information from the vnet and set up vnet { - name: "Add endpoints second half", + name: "Add endpoints get vnet veth mac address", client: &NativeEndpointClient{ eth0VethName: "eth0", vlanVethName: "eth0.1", @@ -210,7 +210,7 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr: false, }, { - name: "Add endpoints fail check eth0.X exists", + name: "Add endpoints fail check vlan veth exists", client: &NativeEndpointClient{ eth0VethName: "eth0", vlanVethName: "eth0.1", @@ -225,7 +225,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "eth0.x doesn't exist: " + netio.ErrMockNetIOFail.Error() + ":eth0.1", + wantErrMsg: "vlan veth doesn't exist: " + netio.ErrMockNetIOFail.Error() + ":eth0.1", }, { name: "Add endpoints fail check vnet veth exists", From e20522c0687c39724e176badde0289a3171acd50 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 14:11:30 -0700 Subject: [PATCH 36/52] Mockns method name enum --- netns/mocknetns.go | 18 +++++++++++++----- network/native_endpointclient_linux_test.go | 10 +++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index 940d2b1cf3..00122be9a0 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -4,6 +4,14 @@ import ( "github.com/pkg/errors" ) +const ( + Get int = 1 + GetFromName int = 2 + Set int = 3 + NewNamed int = 4 + DeleteNamed int = 5 +) + var ErrorMock = errors.New("mock netns error") func newErrorMock(errStr string) error { @@ -23,31 +31,31 @@ func NewMock(failMethod int, failMessage string) *MockNetns { } func (f *MockNetns) Get() (int, error) { - if f.failMethod == 1 { + if f.failMethod == Get { return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) GetFromName(name string) (int, error) { - if f.failMethod == 2 { + if f.failMethod == GetFromName { return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) Set(handle int) error { - if f.failMethod == 3 { + if f.failMethod == Set { return newErrorMock(f.failMessage) } return nil } func (f *MockNetns) NewNamed(name string) (int, error) { - if f.failMethod == 4 { + if f.failMethod == NewNamed { return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) DeleteNamed(name string) error { - if f.failMethod == 5 { + if f.failMethod == DeleteNamed { return newErrorMock(f.failMessage) } return nil diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index da708bae2d..cdce9bfb43 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -35,7 +35,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(2, "no such file or directory"), + netnsClient: netns.NewMock(netns.GetFromName, "no such file or directory"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -123,7 +123,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(1, "netns failure"), + netnsClient: netns.NewMock(netns.Get, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -141,7 +141,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(2, "netns failure"), + netnsClient: netns.NewMock(netns.GetFromName, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -159,7 +159,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(3, "netns failure"), + netnsClient: netns.NewMock(netns.Set, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -304,7 +304,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(5, "netns failure"), + netnsClient: netns.NewMock(netns.DeleteNamed, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), From 41abffaff491872b8aa53bec5ef279eb04d697c3 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 15:20:59 -0700 Subject: [PATCH 37/52] Unable to use GetNetworkInterfaceByName due to NS If I use GetNetworkInterface, I need to be in the vnet NS, but that means I will need to call ExecuteInNS, which causes tests to fail. --- network/native_endpointclient_linux.go | 28 ++++++++++---------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index b5c359d3a0..ef901591bd 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -11,7 +11,6 @@ import ( "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" "github.com/pkg/errors" - vishnetlink "github.com/vishvananda/netlink" ) @@ -89,13 +88,15 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { } log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) - linkAttrs := vishnetlink.NewLinkAttrs() - linkAttrs.Name = client.vlanVethName + // Get parent interface index. Index is consistent across libraries. eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) if err != nil { return errors.Wrap(err, "failed to get eth0 interface") } + + linkAttrs := vishnetlink.NewLinkAttrs() + linkAttrs.Name = client.vlanVethName // Set the peer linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ @@ -103,20 +104,10 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { VlanId: client.vlanID, } log.Printf("[native] Attempting to create %s link in VM NS", client.vlanVethName) - // Attempting to create vlan veth + // Create vlan veth existingErr = vishnetlink.LinkAdd(link) - vlanVethCreated := true - if existingErr != nil { - log.Printf("[native] Existing response is: %s", existingErr.Error()) - if !strings.Contains(strings.ToLower(existingErr.Error()), "file exists") { - // Something else went wrong - return errors.Wrap(err, "error other than vlan veth already exists") - } - // The interface exists already, which is okay - log.Printf("[native] %s already exists", client.vlanVethName) - vlanVethCreated = false - } - if vlanVethCreated { + if existingErr == nil { + // vlan veth was created successfully, so move the vlan veth you created log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) if err = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { @@ -124,6 +115,9 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { } return errors.Wrap(err, "deleting vlan veth in vm ns due to addendpoint failure") } + } else { + // Otherwise, no need to create the vlan veth nor move it anywhere + log.Printf("[native] %s already exists", client.vlanVethName) } if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { @@ -242,7 +236,7 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo if err = client.AddDefaultArp(client.vlanVethName, azureMac); err != nil { return errors.Wrap(err, "failed vnet ns add default arp entry (idempotent)") } - if err := addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { + if err = addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return errors.Wrap(err, "failed adding routes to vnet specific to this container") } // Return to ConfigureContainerInterfacesAndRoutes From 8f3f2abbcb373c5f07685dce181401a39ef4c961 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 19 Jul 2022 15:34:09 -0700 Subject: [PATCH 38/52] Fixes from linter --- netns/netns.go | 15 +++++++++------ network/native_endpointclient_linux.go | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/netns/netns.go b/netns/netns.go index 1389db124b..86b80a73e2 100644 --- a/netns/netns.go +++ b/netns/netns.go @@ -1,6 +1,9 @@ package netns -import "github.com/vishvananda/netns" +import ( + "github.com/pkg/errors" + "github.com/vishvananda/netns" +) type Netns struct{} @@ -10,19 +13,19 @@ func New() *Netns { func (f *Netns) Get() (int, error) { nsHandle, err := netns.Get() - return int(nsHandle), err + return int(nsHandle), errors.Wrap(err, "netns impl") } func (f *Netns) GetFromName(name string) (int, error) { nsHandle, err := netns.GetFromName(name) - return int(nsHandle), err + return int(nsHandle), errors.Wrap(err, "netns impl") } func (f *Netns) Set(fileDescriptor int) error { - return netns.Set(netns.NsHandle(fileDescriptor)) + return errors.Wrap(netns.Set(netns.NsHandle(fileDescriptor)), "netns impl") } func (f *Netns) NewNamed(name string) (int, error) { nsHandle, err := netns.NewNamed(name) - return int(nsHandle), err + return int(nsHandle), errors.Wrap(err, "netns impl") } func (f *Netns) DeleteNamed(name string) error { - return netns.DeleteNamed(name) + return errors.Wrap(netns.DeleteNamed(name), "netns impl") } diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index ef901591bd..6c573b0c1a 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -246,7 +246,7 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo // Helper that gets the routes in the vnet NS for a particular list of IP addresses // Example: 192.168.0.4 dev proto static func (client *NativeEndpointClient) GetVnetRoutes(ipAddresses []net.IPNet) []RouteInfo { - var routeInfoList []RouteInfo + routeInfoList := make([]RouteInfo, 0, len(ipAddresses)) // Add route specifying which device the pod ip(s) are on for _, ipAddr := range ipAddresses { var ( @@ -306,7 +306,7 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) virtualGwNet.String(), destMac) hardwareAddr, err := net.ParseMAC(destMac) if err != nil { - return err + return errors.Wrap(err, "unable to parse mac") } if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, interfaceName, From b5a0621c05e86d9811b150af4548f38adae73079 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 11:02:57 -0700 Subject: [PATCH 39/52] Assume if NS exists, vlan veth exists Tests ok --- netns/mocknetns.go | 14 +-- network/native_endpointclient_linux.go | 95 +++++++++++---------- network/native_endpointclient_linux_test.go | 63 +++++--------- 3 files changed, 81 insertions(+), 91 deletions(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index 00122be9a0..5f4eed6843 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -20,42 +20,44 @@ func newErrorMock(errStr string) error { type MockNetns struct { failMethod int + failMethod2 int failMessage string } -func NewMock(failMethod int, failMessage string) *MockNetns { +func NewMock(failMethod int, failMethod2 int, failMessage string) *MockNetns { return &MockNetns{ failMethod: failMethod, + failMethod2: failMethod2, failMessage: failMessage, } } func (f *MockNetns) Get() (int, error) { - if f.failMethod == Get { + if f.failMethod == Get || f.failMethod2 == Get { return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) GetFromName(name string) (int, error) { - if f.failMethod == GetFromName { + if f.failMethod == GetFromName || f.failMethod2 == GetFromName { return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) Set(handle int) error { - if f.failMethod == Set { + if f.failMethod == Set || f.failMethod2 == Set { return newErrorMock(f.failMessage) } return nil } func (f *MockNetns) NewNamed(name string) (int, error) { - if f.failMethod == NewNamed { + if f.failMethod == NewNamed || f.failMethod2 == NewNamed { return 0, newErrorMock(f.failMessage) } return 1, nil } func (f *MockNetns) DeleteNamed(name string) error { - if f.failMethod == DeleteNamed { + if f.failMethod == DeleteNamed || f.failMethod2 == DeleteNamed { return newErrorMock(f.failMessage) } return nil diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 6c573b0c1a..b4e6e695e3 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -3,7 +3,6 @@ package network import ( "fmt" "net" - "strings" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" @@ -64,61 +63,67 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { log.Printf("[native] Checking if NS exists...") vnetNS, existingErr := client.netnsClient.GetFromName(client.vnetNSName) // If the ns does not exist, the below code will trigger to create it + // This will also (we assume) mean the vlan veth does not exist if existingErr != nil { - log.Printf("[native] Existing response is: %s", existingErr.Error()) - if !strings.Contains(strings.ToLower(existingErr.Error()), "no such file or directory") { - // Something else went wrong - return errors.Wrap(existingErr, "error other than vnet ns doesn't exist") - } - // The vnet ns does not exist, which is okay + // We assume the only possible error is that the namespace doesn't exist log.Printf("[native] No existing NS detected. Creating the vnet namespace and switching to it") vnetNS, err = client.netnsClient.NewNamed(client.vnetNSName) if err != nil { return errors.Wrap(err, "failed to create vnet ns") } + client.vnetNSFileDescriptor = vnetNS + deleteNSIfNotNilErr := client.netnsClient.Set(vmNS) + // Any failure will trigger removing the namespace created + defer func() { + if deleteNSIfNotNilErr != nil { + log.Logf("[native] Removing vnet ns due to failure...") + err = client.netnsClient.DeleteNamed(client.vnetNSName) + if err != nil { + log.Errorf("failed to cleanup/delete ns after failing to create vlan veth") + } + } + }() + if deleteNSIfNotNilErr != nil { + return errors.Wrap(deleteNSIfNotNilErr, "failed to set current ns to vm") + } - } else { - log.Printf("[native] Existing NS detected.") - } - client.vnetNSFileDescriptor = vnetNS - - err = client.netnsClient.Set(vmNS) - if err != nil { - return errors.Wrap(err, "failed to set current ns to vm") - } - - log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) - - // Get parent interface index. Index is consistent across libraries. - eth0, err := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) - if err != nil { - return errors.Wrap(err, "failed to get eth0 interface") - } - - linkAttrs := vishnetlink.NewLinkAttrs() - linkAttrs.Name = client.vlanVethName - // Set the peer - linkAttrs.ParentIndex = eth0.Index - link := &vishnetlink.Vlan{ - LinkAttrs: linkAttrs, - VlanId: client.vlanID, - } - log.Printf("[native] Attempting to create %s link in VM NS", client.vlanVethName) - // Create vlan veth - existingErr = vishnetlink.LinkAdd(link) - if existingErr == nil { - // vlan veth was created successfully, so move the vlan veth you created - log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) - if err = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { - if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { - log.Errorf("deleting vlan veth failed on addendpoint failure:%v", delErr) + // Now create vlan veth + log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) + // Get parent interface index. Index is consistent across libraries. + eth0, deleteNSIfNotNilErr := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) + if deleteNSIfNotNilErr != nil { + return errors.Wrap(deleteNSIfNotNilErr, "failed to get eth0 interface") + } + linkAttrs := vishnetlink.NewLinkAttrs() + linkAttrs.Name = client.vlanVethName + // Set the peer + linkAttrs.ParentIndex = eth0.Index + link := &vishnetlink.Vlan{ + LinkAttrs: linkAttrs, + VlanId: client.vlanID, + } + log.Printf("[native] Attempting to create %s link in VM NS", client.vlanVethName) + // Create vlan veth + deleteNSIfNotNilErr = vishnetlink.LinkAdd(link) + if deleteNSIfNotNilErr == nil { + // vlan veth was created successfully, so move the vlan veth you created + log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) + deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) + if deleteNSIfNotNilErr != nil { + if delErr := client.netlink.DeleteLink(client.vlanVethName); delErr != nil { + log.Errorf("deleting vlan veth failed on addendpoint failure") + } + return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") } - return errors.Wrap(err, "deleting vlan veth in vm ns due to addendpoint failure") + } else { + // Any other error + return errors.Wrap(deleteNSIfNotNilErr, "failed to create vlan vnet link after making new ns") } + } else { - // Otherwise, no need to create the vlan veth nor move it anywhere - log.Printf("[native] %s already exists", client.vlanVethName) + log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanVethName) } + client.vnetNSFileDescriptor = vnetNS if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName); err != nil { return errors.Wrap(err, "failed to create veth pair") diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index cdce9bfb43..9aa2fcbd4e 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -28,21 +28,22 @@ func TestNativeAddEndpoints(t *testing.T) { }{ // Populating VM with data and creating interfaces/links { - name: "Add endpoints no existing vnet ns", + name: "Add endpoints create vnet ns failure", client: &NativeEndpointClient{ eth0VethName: "eth0", vlanVethName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.GetFromName, "no such file or directory"), + netnsClient: netns.NewMock(netns.GetFromName, netns.NewNamed, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), }, - epInfo: &EndpointInfo{}, - wantErr: false, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "failed to create vnet ns: netns failure: " + netns.ErrorMock.Error(), }, { name: "Add endpoints with existing vnet ns", @@ -52,7 +53,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -69,7 +70,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -87,7 +88,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(netns.GetFromName, 0, "netns fail"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -105,11 +106,11 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 2), + netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, wantErr: true, @@ -123,7 +124,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.Get, "netns failure"), + netnsClient: netns.NewMock(netns.Get, 0, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -133,24 +134,6 @@ func TestNativeAddEndpoints(t *testing.T) { wantErr: true, wantErrMsg: "failed to get vm ns handle: netns failure: " + netns.ErrorMock.Error(), }, - { - name: "Add endpoints NetNS GetFromName fail (with error other than file does not exists)", - client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanVethName: "eth0.1", - vnetVethName: "A1veth0", - containerVethName: "B1veth0", - vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.GetFromName, "netns failure"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), - }, - epInfo: &EndpointInfo{}, - wantErr: true, - wantErrMsg: "error other than vnet ns doesn't exist: netns failure: " + netns.ErrorMock.Error(), - }, { name: "Add endpoints NetNS Set fail", client: &NativeEndpointClient{ @@ -159,7 +142,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.Set, "netns failure"), + netnsClient: netns.NewMock(netns.Set, netns.GetFromName, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -200,7 +183,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -217,7 +200,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -235,7 +218,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -279,7 +262,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -304,7 +287,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.DeleteNamed, "netns failure"), + netnsClient: netns.NewMock(netns.DeleteNamed, 0, "netns failure"), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -359,7 +342,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -384,7 +367,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -417,7 +400,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -443,7 +426,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -490,7 +473,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -516,7 +499,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, ""), + netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), From 0726bfd2075bc2e83ae48bed07959ec98be16f1f Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 12:05:53 -0700 Subject: [PATCH 40/52] Fixes for Linter --- netns/mocknetns.go | 6 +++++- netns/netns.go | 4 ++++ network/hnswrapper/hnsv1wrapper.go | 2 +- network/hnswrapper/hnsv1wrapperinterface.go | 2 +- network/hnswrapper/hnsv2wrapperfake.go | 2 +- network/native_endpointclient_linux.go | 7 +++++-- network/native_endpointclient_linux_test.go | 1 + 7 files changed, 18 insertions(+), 6 deletions(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index 5f4eed6843..f9a00821f9 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -24,7 +24,7 @@ type MockNetns struct { failMessage string } -func NewMock(failMethod int, failMethod2 int, failMessage string) *MockNetns { +func NewMock(failMethod, failMethod2 int, failMessage string) *MockNetns { return &MockNetns{ failMethod: failMethod, failMethod2: failMethod2, @@ -38,24 +38,28 @@ func (f *MockNetns) Get() (int, error) { } return 1, nil } + func (f *MockNetns) GetFromName(name string) (int, error) { if f.failMethod == GetFromName || f.failMethod2 == GetFromName { return 0, newErrorMock(f.failMessage) } return 1, nil } + func (f *MockNetns) Set(handle int) error { if f.failMethod == Set || f.failMethod2 == Set { return newErrorMock(f.failMessage) } return nil } + func (f *MockNetns) NewNamed(name string) (int, error) { if f.failMethod == NewNamed || f.failMethod2 == NewNamed { return 0, newErrorMock(f.failMessage) } return 1, nil } + func (f *MockNetns) DeleteNamed(name string) error { if f.failMethod == DeleteNamed || f.failMethod2 == DeleteNamed { return newErrorMock(f.failMessage) diff --git a/netns/netns.go b/netns/netns.go index 86b80a73e2..7937d02317 100644 --- a/netns/netns.go +++ b/netns/netns.go @@ -15,17 +15,21 @@ func (f *Netns) Get() (int, error) { nsHandle, err := netns.Get() return int(nsHandle), errors.Wrap(err, "netns impl") } + func (f *Netns) GetFromName(name string) (int, error) { nsHandle, err := netns.GetFromName(name) return int(nsHandle), errors.Wrap(err, "netns impl") } + func (f *Netns) Set(fileDescriptor int) error { return errors.Wrap(netns.Set(netns.NsHandle(fileDescriptor)), "netns impl") } + func (f *Netns) NewNamed(name string) (int, error) { nsHandle, err := netns.NewNamed(name) return int(nsHandle), errors.Wrap(err, "netns impl") } + func (f *Netns) DeleteNamed(name string) error { return errors.Wrap(netns.DeleteNamed(name), "netns impl") } diff --git a/network/hnswrapper/hnsv1wrapper.go b/network/hnswrapper/hnsv1wrapper.go index 247024220e..55c117347f 100644 --- a/network/hnswrapper/hnsv1wrapper.go +++ b/network/hnswrapper/hnsv1wrapper.go @@ -1,5 +1,5 @@ //go:build windows -//+build windows +// +build windows package hnswrapper diff --git a/network/hnswrapper/hnsv1wrapperinterface.go b/network/hnswrapper/hnsv1wrapperinterface.go index 6f7a3bda45..6367bead28 100644 --- a/network/hnswrapper/hnsv1wrapperinterface.go +++ b/network/hnswrapper/hnsv1wrapperinterface.go @@ -19,5 +19,5 @@ type HnsV1WrapperInterface interface { GetHNSEndpointByID(endpointID string) (*hcsshim.HNSEndpoint, error) HotAttachEndpoint(containerID string, endpointID string) error IsAttached(hnsep *hcsshim.HNSEndpoint, containerID string) (bool, error) - GetHNSGlobals() (*hcsshim.HNSGlobals, error) + GetHNSGlobals() (*hcsshim.HNSGlobals, error) } diff --git a/network/hnswrapper/hnsv2wrapperfake.go b/network/hnswrapper/hnsv2wrapperfake.go index 8e887812c4..4f59aa5aec 100644 --- a/network/hnswrapper/hnsv2wrapperfake.go +++ b/network/hnswrapper/hnsv2wrapperfake.go @@ -42,7 +42,7 @@ func NewHnsv2wrapperFake() *Hnsv2wrapperFake { } } -func delayHnsCall(delay time.Duration){ +func delayHnsCall(delay time.Duration) { time.Sleep(delay) } diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index b4e6e695e3..8edadfe8eb 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -157,6 +157,7 @@ func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { client.vnetMac = vnetVethIf.HardwareAddr return nil } + func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { // There are no rules to add here // Described as rules on ip addresses on the container interface @@ -167,12 +168,14 @@ func (client *NativeEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error func (client *NativeEndpointClient) DeleteEndpointRules(ep *endpoint) { // Never added any endpoint rules } + func (client *NativeEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { return errors.Wrap(err, "failed to move endpoint to container ns") } return nil } + func (client *NativeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { if err := client.netUtilsClient.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return errors.Wrap(err, "failed to setup container interface") @@ -198,7 +201,6 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Container func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { - if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return errors.Wrap(err, "failed to assign ips to container veth interface") } @@ -226,7 +228,6 @@ func (client *NativeEndpointClient) ConfigureContainerInterfacesAndRoutesImpl(ep // Called from ConfigureContainerInterfacesAndRoutes, Namespace: Vnet func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo *EndpointInfo) error { - err := client.netlink.SetLinkState(loopbackIf, true) if err != nil { return errors.Wrap(err, "failed to set loopback link state to up") @@ -322,11 +323,13 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) } return nil } + func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { return ExecuteInNS(client.vnetNSName, func() error { return client.DeleteEndpointsImpl(ep) }) } + func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 9aa2fcbd4e..c17ff6a22b 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -243,6 +243,7 @@ func TestNativeAddEndpoints(t *testing.T) { }) } } + func TestNativeDeleteEndpoints(t *testing.T) { nl := netlink.NewMockNetlink(false, "") plc := platform.NewMockExecClient(false) From 50d1f2bffe6b6539b443ed8e5d4e14a4139e03e4 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 13:19:56 -0700 Subject: [PATCH 41/52] Fix delete tests --- network/native_endpointclient_linux.go | 18 ++++++----- network/native_endpointclient_linux_test.go | 35 ++++++++++++++++++--- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 8edadfe8eb..4c9525db6d 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -326,22 +326,24 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { return ExecuteInNS(client.vnetNSName, func() error { - return client.DeleteEndpointsImpl(ep) + routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) + if err != nil { + return errors.Wrap(err, ("failed to get route list")) + } + log.Printf("[native] Routes remaining: %v", routes) + + return client.DeleteEndpointsImpl(ep, len(routes)) }) } -func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint) error { +func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint, routesLeft int) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return errors.Wrap(err, "failed to remove routes") } + log.Printf("[native] There are %d routes remaining", routesLeft) - routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) - if err != nil { - return errors.Wrap(err, ("failed to get route list")) - } - log.Printf("[native] There are %d routes remaining: %v", len(routes), routes) - if len(routes) <= numDefaultRoutes { + if routesLeft <= numDefaultRoutes { // Deletes default arp, default routes, vlan veth; there are two default routes // so when we have <= numDefaultRoutes routes left, no containers use this namespace log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index c17ff6a22b..a6b8d471c8 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -254,9 +254,10 @@ func TestNativeDeleteEndpoints(t *testing.T) { ep *endpoint wantErr bool wantErrMsg string + routesLeft int }{ { - name: "Delete endpoint good path", + name: "Delete endpoint delete vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", vlanVethName: "eth0.1", @@ -277,9 +278,34 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, }, }, - wantErr: false, + routesLeft: numDefaultRoutes, + wantErr: false, + }, + { + name: "Delete endpoint do not delete vnet ns it is still in use", + client: &NativeEndpointClient{ + eth0VethName: "eth0", + vlanVethName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: netns.NewMock(netns.DeleteNamed, 0, "netns failure"), + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + ep: &endpoint{ + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + }, + routesLeft: numDefaultRoutes + 1, + wantErr: false, }, - // You must have <= 2 ip routes on your machine for this to pass { name: "Delete endpoint fail to delete namespace", client: &NativeEndpointClient{ @@ -302,6 +328,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { }, }, }, + routesLeft: numDefaultRoutes, wantErr: true, wantErrMsg: "failed to delete namespace: netns failure: " + netns.ErrorMock.Error(), }, @@ -310,7 +337,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.DeleteEndpointsImpl(tt.ep) + err := tt.client.DeleteEndpointsImpl(tt.ep, tt.routesLeft) if tt.wantErr { require.Error(t, err) require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) From fe00e3a520e754d97fd465a50c56f47c84cab6f1 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 13:41:35 -0700 Subject: [PATCH 42/52] Fix delete tests bug --- network/native_endpointclient_linux.go | 6 ++-- network/native_endpointclient_linux_test.go | 38 +++++++++------------ 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 4c9525db6d..d2cec73b7e 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -338,12 +338,12 @@ func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint, routesLeft int) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) + log.Printf("[native] There are %d routes remaining", routesLeft) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return errors.Wrap(err, "failed to remove routes") } - log.Printf("[native] There are %d routes remaining", routesLeft) - - if routesLeft <= numDefaultRoutes { + // We get the # of routes, THEN delete, and see if it adds up + if routesLeft <= numDefaultRoutes+len(routeInfoList) { // Deletes default arp, default routes, vlan veth; there are two default routes // so when we have <= numDefaultRoutes routes left, no containers use this namespace log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index a6b8d471c8..7ce99cb888 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -247,7 +247,16 @@ func TestNativeAddEndpoints(t *testing.T) { func TestNativeDeleteEndpoints(t *testing.T) { nl := netlink.NewMockNetlink(false, "") plc := platform.NewMockExecClient(false) - + IPAddresses := []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + { + IP: net.ParseIP("192.168.0.6"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + } tests := []struct { name string client *NativeEndpointClient @@ -271,14 +280,9 @@ func TestNativeDeleteEndpoints(t *testing.T) { netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ - IPAddresses: []net.IPNet{ - { - IP: net.ParseIP("192.168.0.4"), - Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), - }, - }, + IPAddresses: IPAddresses, }, - routesLeft: numDefaultRoutes, + routesLeft: numDefaultRoutes + len(IPAddresses), wantErr: false, }, { @@ -296,14 +300,9 @@ func TestNativeDeleteEndpoints(t *testing.T) { netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ - IPAddresses: []net.IPNet{ - { - IP: net.ParseIP("192.168.0.4"), - Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), - }, - }, + IPAddresses: IPAddresses, }, - routesLeft: numDefaultRoutes + 1, + routesLeft: numDefaultRoutes + len(IPAddresses) + 1, wantErr: false, }, { @@ -321,14 +320,9 @@ func TestNativeDeleteEndpoints(t *testing.T) { netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ - IPAddresses: []net.IPNet{ - { - IP: net.ParseIP("192.168.0.4"), - Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), - }, - }, + IPAddresses: IPAddresses, }, - routesLeft: numDefaultRoutes, + routesLeft: numDefaultRoutes + len(IPAddresses), wantErr: true, wantErrMsg: "failed to delete namespace: netns failure: " + netns.ErrorMock.Error(), }, From 60ce0703e1932fa7473a029e88529d6368b52af5 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 20 Jul 2022 15:32:20 -0700 Subject: [PATCH 43/52] Go mod tidy for linting Hopefully this fixes the windows lint error --- go.mod | 2 +- go.sum | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 44a75cbde0..f5b336d688 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,7 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect github.com/vishvananda/netlink v1.2.1-beta.2 - github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect + github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.6.0 // indirect diff --git a/go.sum b/go.sum index 60a2b412a3..92c01e753c 100644 --- a/go.sum +++ b/go.sum @@ -830,15 +830,12 @@ github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg= github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= From f31e59f0c2e2d8432230cc737a823c83109afd00 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 10:56:21 -0700 Subject: [PATCH 44/52] No lint on vishvananda netns Maybe this will fix the windows linter? --- netns/netns.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/netns/netns.go b/netns/netns.go index 7937d02317..c5fc351e2b 100644 --- a/netns/netns.go +++ b/netns/netns.go @@ -1,7 +1,9 @@ +//nolint package netns import ( "github.com/pkg/errors" + //nolint "github.com/vishvananda/netns" ) From fd3341be8f3b9dcd077d27525a02684fbc0c2c61 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 11:24:59 -0700 Subject: [PATCH 45/52] Build linux only for netns package Maybe this fixes the linter error? --- netns/mocknetns.go | 3 +++ netns/netns.go | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/netns/mocknetns.go b/netns/mocknetns.go index f9a00821f9..432fe03258 100644 --- a/netns/mocknetns.go +++ b/netns/mocknetns.go @@ -1,3 +1,6 @@ +//go:build linux +// +build linux + package netns import ( diff --git a/netns/netns.go b/netns/netns.go index c5fc351e2b..11a48a4be1 100644 --- a/netns/netns.go +++ b/netns/netns.go @@ -1,9 +1,11 @@ +//go:build linux +// +build linux + //nolint package netns import ( "github.com/pkg/errors" - //nolint "github.com/vishvananda/netns" ) From 92cff30935edce6f8199ac580afe201d587db35b Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 21 Jul 2022 11:34:43 -0700 Subject: [PATCH 46/52] Remove nolint to see if linter fails --- netns/netns.go | 1 - 1 file changed, 1 deletion(-) diff --git a/netns/netns.go b/netns/netns.go index 11a48a4be1..dba8f30f80 100644 --- a/netns/netns.go +++ b/netns/netns.go @@ -1,7 +1,6 @@ //go:build linux // +build linux -//nolint package netns import ( From d344df6f7d768b5604b09dd8621b970491cae076 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Tue, 26 Jul 2022 09:43:51 -0700 Subject: [PATCH 47/52] Moved netns interface to caller, generalized tests Tests ok, Native ok --- netns/mocknetns.go | 71 ------ network/native_endpointclient_linux.go | 9 +- network/native_endpointclient_linux_test.go | 231 ++++++++++++++------ network/netnsinterface.go | 9 - 4 files changed, 175 insertions(+), 145 deletions(-) delete mode 100644 netns/mocknetns.go delete mode 100644 network/netnsinterface.go diff --git a/netns/mocknetns.go b/netns/mocknetns.go deleted file mode 100644 index 432fe03258..0000000000 --- a/netns/mocknetns.go +++ /dev/null @@ -1,71 +0,0 @@ -//go:build linux -// +build linux - -package netns - -import ( - "github.com/pkg/errors" -) - -const ( - Get int = 1 - GetFromName int = 2 - Set int = 3 - NewNamed int = 4 - DeleteNamed int = 5 -) - -var ErrorMock = errors.New("mock netns error") - -func newErrorMock(errStr string) error { - return errors.Wrap(ErrorMock, errStr) -} - -type MockNetns struct { - failMethod int - failMethod2 int - failMessage string -} - -func NewMock(failMethod, failMethod2 int, failMessage string) *MockNetns { - return &MockNetns{ - failMethod: failMethod, - failMethod2: failMethod2, - failMessage: failMessage, - } -} - -func (f *MockNetns) Get() (int, error) { - if f.failMethod == Get || f.failMethod2 == Get { - return 0, newErrorMock(f.failMessage) - } - return 1, nil -} - -func (f *MockNetns) GetFromName(name string) (int, error) { - if f.failMethod == GetFromName || f.failMethod2 == GetFromName { - return 0, newErrorMock(f.failMessage) - } - return 1, nil -} - -func (f *MockNetns) Set(handle int) error { - if f.failMethod == Set || f.failMethod2 == Set { - return newErrorMock(f.failMessage) - } - return nil -} - -func (f *MockNetns) NewNamed(name string) (int, error) { - if f.failMethod == NewNamed || f.failMethod2 == NewNamed { - return 0, newErrorMock(f.failMessage) - } - return 1, nil -} - -func (f *MockNetns) DeleteNamed(name string) error { - if f.failMethod == DeleteNamed || f.failMethod2 == DeleteNamed { - return newErrorMock(f.failMessage) - } - return nil -} diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index d2cec73b7e..bc674ca8fd 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -19,6 +19,13 @@ const ( numDefaultRoutes = 2 // VNET NS, when no containers use it, has this many routes ) +type netnsClient interface { + Get() (fileDescriptor int, err error) + GetFromName(name string) (fileDescriptor int, err error) + Set(fileDescriptor int) (err error) + NewNamed(name string) (fileDescriptor int, err error) + DeleteNamed(name string) (err error) +} type NativeEndpointClient struct { eth0VethName string // So like eth0 vlanVethName string // So like eth0.1 @@ -33,7 +40,7 @@ type NativeEndpointClient struct { nw *network vlanID int - netnsClient NetnsInterface + netnsClient netnsClient netlink netlink.NetlinkInterface netioshim netio.NetIOInterface plClient platform.ExecClient diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 7ce99cb888..615737e16b 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -9,12 +9,66 @@ import ( "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" + "github.com/pkg/errors" "github.com/stretchr/testify/require" ) +var ErrorMock = errors.New("mock netns error") + +func newErrorMock(errStr string) error { + return errors.Wrap(ErrorMock, errStr) +} + +type mockNetns struct { + get func() (fileDescriptor int, err error) + getFromName func(name string) (fileDescriptor int, err error) + set func(fileDescriptor int) (err error) + newNamed func(name string) (fileDescriptor int, err error) + deleteNamed func(name string) (err error) +} + +func (netns *mockNetns) Get() (fileDescriptor int, err error) { + return netns.get() +} + +func (netns *mockNetns) GetFromName(name string) (fileDescriptor int, err error) { + return netns.getFromName(name) +} + +func (netns *mockNetns) Set(fileDescriptor int) (err error) { + return netns.set(fileDescriptor) +} + +func (netns *mockNetns) NewNamed(name string) (fileDescriptor int, err error) { + return netns.newNamed(name) +} + +func (netns *mockNetns) DeleteNamed(name string) (err error) { + return netns.deleteNamed(name) +} + +func defaultGet() (int, error) { + return 1, nil +} + +func defaultGetFromName(name string) (int, error) { + return 1, nil +} + +func defaultSet(handle int) error { + return nil +} + +func defaultNewNamed(name string) (int, error) { + return 1, nil +} + +func defaultDeleteNamed(name string) error { + return nil +} + func TestNativeAddEndpoints(t *testing.T) { nl := netlink.NewMockNetlink(false, "") plc := platform.NewMockExecClient(false) @@ -35,15 +89,23 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.GetFromName, netns.NewNamed, "netns failure"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: func(name string) (fileDescriptor int, err error) { + return 0, newErrorMock("netns failure") + }, + newNamed: func(name string) (fileDescriptor int, err error) { + return 0, newErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to create vnet ns: netns failure: " + netns.ErrorMock.Error(), + wantErrMsg: "failed to create vnet ns: netns failure: " + ErrorMock.Error(), }, { name: "Add endpoints with existing vnet ns", @@ -53,11 +115,17 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: defaultGetFromName, + newNamed: defaultNewNamed, + set: defaultSet, + deleteNamed: defaultDeleteNamed, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, wantErr: false, @@ -70,11 +138,17 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), - netlink: netlink.NewMockNetlink(true, "netlink fail"), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: defaultGetFromName, + newNamed: defaultNewNamed, + set: defaultSet, + deleteNamed: defaultDeleteNamed, + }, + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, wantErr: true, @@ -88,11 +162,19 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.GetFromName, 0, "netns fail"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 1), + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: func(name string) (fileDescriptor int, err error) { + return 0, newErrorMock("netns failure") + }, + newNamed: defaultNewNamed, + set: defaultSet, + deleteNamed: defaultDeleteNamed, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, wantErr: true, @@ -106,11 +188,17 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 1), + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: defaultGetFromName, + newNamed: defaultNewNamed, + set: defaultSet, + deleteNamed: defaultDeleteNamed, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 1), }, epInfo: &EndpointInfo{}, wantErr: true, @@ -124,15 +212,19 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.Get, 0, "netns failure"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + get: func() (fileDescriptor int, err error) { + return 0, newErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to get vm ns handle: netns failure: " + netns.ErrorMock.Error(), + wantErrMsg: "failed to get vm ns handle: netns failure: " + ErrorMock.Error(), }, { name: "Add endpoints NetNS Set fail", @@ -142,15 +234,25 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.Set, netns.GetFromName, "netns failure"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: func(name string) (fileDescriptor int, err error) { + return 0, newErrorMock("do not fail on this error") + }, + newNamed: defaultNewNamed, + set: func(fileDescriptor int) (err error) { + return newErrorMock("netns failure") + }, + deleteNamed: defaultDeleteNamed, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to set current ns to vm: netns failure: " + netns.ErrorMock.Error(), + wantErrMsg: "failed to set current ns to vm: netns failure: " + ErrorMock.Error(), }, } @@ -183,7 +285,6 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -200,7 +301,6 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -218,7 +318,6 @@ func TestNativeAddEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -273,11 +372,13 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(0, 0, ""), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + deleteNamed: defaultDeleteNamed, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ IPAddresses: IPAddresses, @@ -293,11 +394,15 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.DeleteNamed, 0, "netns failure"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + deleteNamed: func(name string) (err error) { + return newErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ IPAddresses: IPAddresses, @@ -313,18 +418,22 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", - netnsClient: netns.NewMock(netns.DeleteNamed, 0, "netns failure"), - netlink: netlink.NewMockNetlink(false, ""), - plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(false, 0), + netnsClient: &mockNetns{ + deleteNamed: func(name string) (err error) { + return newErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ IPAddresses: IPAddresses, }, routesLeft: numDefaultRoutes + len(IPAddresses), wantErr: true, - wantErrMsg: "failed to delete namespace: netns failure: " + netns.ErrorMock.Error(), + wantErrMsg: "failed to delete namespace: netns failure: " + ErrorMock.Error(), }, } @@ -364,7 +473,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -389,7 +497,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -422,7 +529,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(true, "netlink fail"), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -448,7 +554,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -495,7 +600,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), @@ -521,7 +625,6 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { containerVethName: "B1veth0", vnetNSName: "az_ns_1", vnetMac: vnetMac, - netnsClient: netns.NewMock(0, 0, ""), netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), diff --git a/network/netnsinterface.go b/network/netnsinterface.go deleted file mode 100644 index 8f6311c386..0000000000 --- a/network/netnsinterface.go +++ /dev/null @@ -1,9 +0,0 @@ -package network - -type NetnsInterface interface { - Get() (fileDescriptor int, err error) - GetFromName(name string) (fileDescriptor int, err error) - Set(fileDescriptor int) (err error) - NewNamed(name string) (fileDescriptor int, err error) - DeleteNamed(name string) (err error) -} From 81c988606de27a494126f522efbc318bac828c1b Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 27 Jul 2022 07:56:53 -0700 Subject: [PATCH 48/52] Typos --- network/native_endpointclient_linux.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index bc674ca8fd..5add1d5966 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -244,7 +244,7 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) if err = client.AddDefaultRoutes(client.vlanVethName); err != nil { - return errors.Wrap(err, "failed vnet ns add default/gateway routes (indempotent)") + return errors.Wrap(err, "failed vnet ns add default/gateway routes (idempotent)") } if err = client.AddDefaultArp(client.vlanVethName, azureMac); err != nil { return errors.Wrap(err, "failed vnet ns add default arp entry (idempotent)") @@ -362,7 +362,7 @@ func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint, routesLeft return nil } -// Helper function that allows executing a function with one parameter in a VM namespace +// Helper function that allows executing a function in a VM namespace // Does not work for process namespaces func ExecuteInNS(nsName string, f func() error) error { // Current namespace From 60bfc32eda984590a7d2fbedfb20bf204053db13 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Wed, 27 Jul 2022 14:31:30 -0700 Subject: [PATCH 49/52] Reordered if statement, unwrapped arp Tests ok, ping ok, wget ok --- network/native_endpointclient_linux.go | 30 +++++++++++--------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 5add1d5966..2f1c637d88 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -112,21 +112,19 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { log.Printf("[native] Attempting to create %s link in VM NS", client.vlanVethName) // Create vlan veth deleteNSIfNotNilErr = vishnetlink.LinkAdd(link) - if deleteNSIfNotNilErr == nil { - // vlan veth was created successfully, so move the vlan veth you created - log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) - deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) - if deleteNSIfNotNilErr != nil { - if delErr := client.netlink.DeleteLink(client.vlanVethName); delErr != nil { - log.Errorf("deleting vlan veth failed on addendpoint failure") - } - return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") - } - } else { - // Any other error + if deleteNSIfNotNilErr != nil { + // Any failure to add the link should error (auto delete NS) return errors.Wrap(deleteNSIfNotNilErr, "failed to create vlan vnet link after making new ns") } - + // vlan veth was created successfully, so move the vlan veth you created + log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) + deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) + if deleteNSIfNotNilErr != nil { + if delErr := client.netlink.DeleteLink(client.vlanVethName); delErr != nil { + log.Errorf("deleting vlan veth failed on addendpoint failure") + } + return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") + } } else { log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanVethName) } @@ -321,11 +319,7 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) if err != nil { return errors.Wrap(err, "unable to parse mac") } - if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, - interfaceName, - virtualGwNet.IP, - hardwareAddr, - false); err != nil { + if err := client.netlink.AddOrRemoveStaticArp(netlink.ADD, interfaceName, virtualGwNet.IP, hardwareAddr, false); err != nil { return fmt.Errorf("adding arp entry failed: %w", err) } return nil From dd066704b86a7402b9d493cf9e958f8622cb92ca Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 1 Aug 2022 11:02:32 -0700 Subject: [PATCH 50/52] Renamed veth, fixed logs --- network/endpoint_linux.go | 4 +-- network/native_endpointclient_linux.go | 24 ++++++------- network/native_endpointclient_linux_test.go | 38 ++++++++++----------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 9dbd21bfec..4c5330bd88 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -97,7 +97,7 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p epClient = &NativeEndpointClient{ eth0VethName: nw.extIf.Name, - vlanVethName: vlanVethName, + vlanEthName: vlanVethName, vnetVethName: hostIfName, containerVethName: contIfName, vnetNSName: vnetNSName, @@ -268,7 +268,7 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. epClient = &NativeEndpointClient{ eth0VethName: nw.extIf.Name, - vlanVethName: vlanVethName, + vlanEthName: vlanVethName, vnetVethName: ep.HostIfName, containerVethName: "", vnetNSName: vnetNSName, diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index 2f1c637d88..ab1c1934c8 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -28,7 +28,7 @@ type netnsClient interface { } type NativeEndpointClient struct { eth0VethName string // So like eth0 - vlanVethName string // So like eth0.1 + vlanEthName string // So like eth0.1 vnetVethName string // Peer is containerVethName containerVethName string // Peer is vnetVethName @@ -102,14 +102,14 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { return errors.Wrap(deleteNSIfNotNilErr, "failed to get eth0 interface") } linkAttrs := vishnetlink.NewLinkAttrs() - linkAttrs.Name = client.vlanVethName + linkAttrs.Name = client.vlanEthName // Set the peer linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ LinkAttrs: linkAttrs, VlanId: client.vlanID, } - log.Printf("[native] Attempting to create %s link in VM NS", client.vlanVethName) + log.Printf("[native] Attempting to create %s link in VM NS", client.vlanEthName) // Create vlan veth deleteNSIfNotNilErr = vishnetlink.LinkAdd(link) if deleteNSIfNotNilErr != nil { @@ -117,16 +117,16 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { return errors.Wrap(deleteNSIfNotNilErr, "failed to create vlan vnet link after making new ns") } // vlan veth was created successfully, so move the vlan veth you created - log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) - deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanVethName, uintptr(client.vnetNSFileDescriptor)) + log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanEthName, uintptr(client.vnetNSFileDescriptor)) + deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanEthName, uintptr(client.vnetNSFileDescriptor)) if deleteNSIfNotNilErr != nil { - if delErr := client.netlink.DeleteLink(client.vlanVethName); delErr != nil { + if delErr := client.netlink.DeleteLink(client.vlanEthName); delErr != nil { log.Errorf("deleting vlan veth failed on addendpoint failure") } return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") } } else { - log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanVethName) + log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanEthName) } client.vnetNSFileDescriptor = vnetNS @@ -151,7 +151,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Called from AddEndpoints, Namespace: Vnet func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { - _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanVethName) + _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanEthName) if err != nil { return errors.Wrap(err, "vlan veth doesn't exist") } @@ -241,10 +241,10 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo // Add route specifying which device the pod ip(s) are on routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) - if err = client.AddDefaultRoutes(client.vlanVethName); err != nil { + if err = client.AddDefaultRoutes(client.vlanEthName); err != nil { return errors.Wrap(err, "failed vnet ns add default/gateway routes (idempotent)") } - if err = client.AddDefaultArp(client.vlanVethName, azureMac); err != nil { + if err = client.AddDefaultArp(client.vlanEthName, azureMac); err != nil { return errors.Wrap(err, "failed vnet ns add default arp entry (idempotent)") } if err = addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { @@ -375,14 +375,14 @@ func ExecuteInNS(nsName string, f func() error) error { } defer ns.Close() // Enter the network namespace - log.Printf("[ExecuteInNS] Entering vnetns %s.", ns.file.Name()) + log.Printf("[ExecuteInNS] Entering ns %s.", ns.file.Name()) if err := ns.Enter(); err != nil { return err } // Exit network namespace defer func() { - log.Printf("[ExecuteInNS] Exiting vnetns %s.", ns.file.Name()) + log.Printf("[ExecuteInNS] Exiting ns %s.", ns.file.Name()) if err := ns.Exit(); err != nil { log.Errorf("[ExecuteInNS] Could not exit ns, err:%v.", err) } diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 615737e16b..cd7f199bd5 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -85,7 +85,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints create vnet ns failure", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -111,7 +111,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints with existing vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -134,7 +134,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints netlink fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -158,7 +158,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints get interface fail for primary interface (eth0)", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -184,7 +184,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints get interface fail for getting container veth", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -208,7 +208,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints NetNS Get fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -230,7 +230,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints NetNS Set fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -281,7 +281,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints get vnet veth mac address", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -297,7 +297,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints fail check vlan veth exists", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -314,7 +314,7 @@ func TestNativeAddEndpoints(t *testing.T) { name: "Add endpoints fail check vnet veth exists", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -368,7 +368,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { name: "Delete endpoint delete vnet ns", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -390,7 +390,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { name: "Delete endpoint do not delete vnet ns it is still in use", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -414,7 +414,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { name: "Delete endpoint fail to delete namespace", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -468,7 +468,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes good path for container", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -492,7 +492,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes multiple IPs", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -524,7 +524,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes assign ip fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -549,7 +549,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes container 2nd default route added fail", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -595,7 +595,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes good path for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -620,7 +620,7 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { name: "Configure interface and routes fail final routes for vnet", client: &NativeEndpointClient{ eth0VethName: "eth0", - vlanVethName: "eth0.1", + vlanEthName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", From 12cc662f479cc7eb34572d2c813715f01f429db6 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 1 Aug 2022 11:38:41 -0700 Subject: [PATCH 51/52] Made deleteEndpoints logic clearer, renamed error --- network/native_endpointclient_linux.go | 28 ++++++++---- network/native_endpointclient_linux_test.go | 49 ++++++++++++--------- 2 files changed, 47 insertions(+), 30 deletions(-) diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index ab1c1934c8..e1f60d2a74 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -327,24 +327,34 @@ func (client *NativeEndpointClient) AddDefaultArp(interfaceName, destMac string) func (client *NativeEndpointClient) DeleteEndpoints(ep *endpoint) error { return ExecuteInNS(client.vnetNSName, func() error { - routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) - if err != nil { - return errors.Wrap(err, ("failed to get route list")) + // Passing in functionality to get number of routes after deletion + getNumRoutesLeft := func() (int, error) { + routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) + if err != nil { + return 0, errors.Wrap(err, "failed to get num routes left") + } + return len(routes), nil } - log.Printf("[native] Routes remaining: %v", routes) - return client.DeleteEndpointsImpl(ep, len(routes)) + return client.DeleteEndpointsImpl(ep, getNumRoutesLeft) }) } -func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint, routesLeft int) error { +// getNumRoutesLeft is a function which gets the current number of routes in the namespace. Namespace: Vnet +func (client *NativeEndpointClient) DeleteEndpointsImpl(ep *endpoint, getNumRoutesLeft func() (int, error)) error { routeInfoList := client.GetVnetRoutes(ep.IPAddresses) - log.Printf("[native] There are %d routes remaining", routesLeft) if err := deleteRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { return errors.Wrap(err, "failed to remove routes") } - // We get the # of routes, THEN delete, and see if it adds up - if routesLeft <= numDefaultRoutes+len(routeInfoList) { + + routesLeft, err := getNumRoutesLeft() + if err != nil { + return err + } + + log.Printf("[native] There are %d routes remaining after deletion", routesLeft) + + if routesLeft <= numDefaultRoutes { // Deletes default arp, default routes, vlan veth; there are two default routes // so when we have <= numDefaultRoutes routes left, no containers use this namespace log.Printf("[native] Deleting namespace %s as no containers occupy it", client.vnetNSName) diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index cd7f199bd5..8bb794b7d3 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -15,10 +15,10 @@ import ( "github.com/stretchr/testify/require" ) -var ErrorMock = errors.New("mock netns error") +var errNetnsMock = errors.New("mock netns error") -func newErrorMock(errStr string) error { - return errors.Wrap(ErrorMock, errStr) +func newNetnsErrorMock(errStr string) error { + return errors.Wrap(errNetnsMock, errStr) } type mockNetns struct { @@ -92,10 +92,10 @@ func TestNativeAddEndpoints(t *testing.T) { netnsClient: &mockNetns{ get: defaultGet, getFromName: func(name string) (fileDescriptor int, err error) { - return 0, newErrorMock("netns failure") + return 0, newNetnsErrorMock("netns failure") }, newNamed: func(name string) (fileDescriptor int, err error) { - return 0, newErrorMock("netns failure") + return 0, newNetnsErrorMock("netns failure") }, }, netlink: netlink.NewMockNetlink(false, ""), @@ -105,7 +105,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to create vnet ns: netns failure: " + ErrorMock.Error(), + wantErrMsg: "failed to create vnet ns: netns failure: " + errNetnsMock.Error(), }, { name: "Add endpoints with existing vnet ns", @@ -165,7 +165,7 @@ func TestNativeAddEndpoints(t *testing.T) { netnsClient: &mockNetns{ get: defaultGet, getFromName: func(name string) (fileDescriptor int, err error) { - return 0, newErrorMock("netns failure") + return 0, newNetnsErrorMock("netns failure") }, newNamed: defaultNewNamed, set: defaultSet, @@ -214,7 +214,7 @@ func TestNativeAddEndpoints(t *testing.T) { vnetNSName: "az_ns_1", netnsClient: &mockNetns{ get: func() (fileDescriptor int, err error) { - return 0, newErrorMock("netns failure") + return 0, newNetnsErrorMock("netns failure") }, }, netlink: netlink.NewMockNetlink(false, ""), @@ -224,7 +224,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to get vm ns handle: netns failure: " + ErrorMock.Error(), + wantErrMsg: "failed to get vm ns handle: netns failure: " + errNetnsMock.Error(), }, { name: "Add endpoints NetNS Set fail", @@ -237,11 +237,11 @@ func TestNativeAddEndpoints(t *testing.T) { netnsClient: &mockNetns{ get: defaultGet, getFromName: func(name string) (fileDescriptor int, err error) { - return 0, newErrorMock("do not fail on this error") + return 0, newNetnsErrorMock("do not fail on this error") }, newNamed: defaultNewNamed, set: func(fileDescriptor int) (err error) { - return newErrorMock("netns failure") + return newNetnsErrorMock("netns failure") }, deleteNamed: defaultDeleteNamed, }, @@ -252,7 +252,7 @@ func TestNativeAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to set current ns to vm: netns failure: " + ErrorMock.Error(), + wantErrMsg: "failed to set current ns to vm: netns failure: " + errNetnsMock.Error(), }, } @@ -356,13 +356,14 @@ func TestNativeDeleteEndpoints(t *testing.T) { Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), }, } + tests := []struct { name string client *NativeEndpointClient ep *endpoint wantErr bool wantErrMsg string - routesLeft int + routesLeft func() (int, error) }{ { name: "Delete endpoint delete vnet ns", @@ -383,8 +384,10 @@ func TestNativeDeleteEndpoints(t *testing.T) { ep: &endpoint{ IPAddresses: IPAddresses, }, - routesLeft: numDefaultRoutes + len(IPAddresses), - wantErr: false, + routesLeft: func() (int, error) { + return numDefaultRoutes, nil + }, + wantErr: false, }, { name: "Delete endpoint do not delete vnet ns it is still in use", @@ -396,7 +399,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetNSName: "az_ns_1", netnsClient: &mockNetns{ deleteNamed: func(name string) (err error) { - return newErrorMock("netns failure") + return newNetnsErrorMock("netns failure") }, }, netlink: netlink.NewMockNetlink(false, ""), @@ -407,8 +410,10 @@ func TestNativeDeleteEndpoints(t *testing.T) { ep: &endpoint{ IPAddresses: IPAddresses, }, - routesLeft: numDefaultRoutes + len(IPAddresses) + 1, - wantErr: false, + routesLeft: func() (int, error) { + return numDefaultRoutes + 1, nil + }, + wantErr: false, }, { name: "Delete endpoint fail to delete namespace", @@ -420,7 +425,7 @@ func TestNativeDeleteEndpoints(t *testing.T) { vnetNSName: "az_ns_1", netnsClient: &mockNetns{ deleteNamed: func(name string) (err error) { - return newErrorMock("netns failure") + return newNetnsErrorMock("netns failure") }, }, netlink: netlink.NewMockNetlink(false, ""), @@ -431,9 +436,11 @@ func TestNativeDeleteEndpoints(t *testing.T) { ep: &endpoint{ IPAddresses: IPAddresses, }, - routesLeft: numDefaultRoutes + len(IPAddresses), + routesLeft: func() (int, error) { + return numDefaultRoutes, nil + }, wantErr: true, - wantErrMsg: "failed to delete namespace: netns failure: " + ErrorMock.Error(), + wantErrMsg: "failed to delete namespace: netns failure: " + errNetnsMock.Error(), }, } From 64794e682c000a072b00196a5cbf9c2416dfa176 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Mon, 1 Aug 2022 14:54:23 -0700 Subject: [PATCH 52/52] Renamed eth0 to primaryHostIfName, vlanEth to vlanIf --- network/endpoint_linux.go | 8 +-- network/native_endpointclient_linux.go | 26 +++---- network/native_endpointclient_linux_test.go | 76 ++++++++++----------- 3 files changed, 55 insertions(+), 55 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 4c5330bd88..0c21b6c0d5 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -96,8 +96,8 @@ func (nw *network) newEndpointImpl(_ apipaClient, nl netlink.NetlinkInterface, p vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) epClient = &NativeEndpointClient{ - eth0VethName: nw.extIf.Name, - vlanEthName: vlanVethName, + primaryHostIfName: nw.extIf.Name, + vlanIfName: vlanVethName, vnetVethName: hostIfName, containerVethName: contIfName, vnetNSName: vnetNSName, @@ -267,8 +267,8 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. vnetNSName := fmt.Sprintf("az_ns_%d", ep.VlanID) epClient = &NativeEndpointClient{ - eth0VethName: nw.extIf.Name, - vlanEthName: vlanVethName, + primaryHostIfName: nw.extIf.Name, + vlanIfName: vlanVethName, vnetVethName: ep.HostIfName, containerVethName: "", vnetNSName: vnetNSName, diff --git a/network/native_endpointclient_linux.go b/network/native_endpointclient_linux.go index e1f60d2a74..8ddd4b6bec 100644 --- a/network/native_endpointclient_linux.go +++ b/network/native_endpointclient_linux.go @@ -27,8 +27,8 @@ type netnsClient interface { DeleteNamed(name string) (err error) } type NativeEndpointClient struct { - eth0VethName string // So like eth0 - vlanEthName string // So like eth0.1 + primaryHostIfName string // So like eth0 + vlanIfName string // So like eth0.1 vnetVethName string // Peer is containerVethName containerVethName string // Peer is vnetVethName @@ -95,21 +95,21 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { } // Now create vlan veth - log.Printf("[native] Create the host vlan link after getting eth0: %s", client.eth0VethName) + log.Printf("[native] Create the host vlan link after getting eth0: %s", client.primaryHostIfName) // Get parent interface index. Index is consistent across libraries. - eth0, deleteNSIfNotNilErr := client.netioshim.GetNetworkInterfaceByName(client.eth0VethName) + eth0, deleteNSIfNotNilErr := client.netioshim.GetNetworkInterfaceByName(client.primaryHostIfName) if deleteNSIfNotNilErr != nil { return errors.Wrap(deleteNSIfNotNilErr, "failed to get eth0 interface") } linkAttrs := vishnetlink.NewLinkAttrs() - linkAttrs.Name = client.vlanEthName + linkAttrs.Name = client.vlanIfName // Set the peer linkAttrs.ParentIndex = eth0.Index link := &vishnetlink.Vlan{ LinkAttrs: linkAttrs, VlanId: client.vlanID, } - log.Printf("[native] Attempting to create %s link in VM NS", client.vlanEthName) + log.Printf("[native] Attempting to create %s link in VM NS", client.vlanIfName) // Create vlan veth deleteNSIfNotNilErr = vishnetlink.LinkAdd(link) if deleteNSIfNotNilErr != nil { @@ -117,16 +117,16 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { return errors.Wrap(deleteNSIfNotNilErr, "failed to create vlan vnet link after making new ns") } // vlan veth was created successfully, so move the vlan veth you created - log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanEthName, uintptr(client.vnetNSFileDescriptor)) - deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanEthName, uintptr(client.vnetNSFileDescriptor)) + log.Printf("[native] Move vlan link (%s) to vnet NS: %d", client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) + deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) if deleteNSIfNotNilErr != nil { - if delErr := client.netlink.DeleteLink(client.vlanEthName); delErr != nil { + if delErr := client.netlink.DeleteLink(client.vlanIfName); delErr != nil { log.Errorf("deleting vlan veth failed on addendpoint failure") } return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") } } else { - log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanEthName) + log.Printf("[native] Existing NS (%s) detected. Assuming %s exists too", client.vnetNSName, client.vlanIfName) } client.vnetNSFileDescriptor = vnetNS @@ -151,7 +151,7 @@ func (client *NativeEndpointClient) PopulateVM(epInfo *EndpointInfo) error { // Called from AddEndpoints, Namespace: Vnet func (client *NativeEndpointClient) PopulateVnet(epInfo *EndpointInfo) error { - _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanEthName) + _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanIfName) if err != nil { return errors.Wrap(err, "vlan veth doesn't exist") } @@ -241,10 +241,10 @@ func (client *NativeEndpointClient) ConfigureVnetInterfacesAndRoutesImpl(epInfo // Add route specifying which device the pod ip(s) are on routeInfoList := client.GetVnetRoutes(epInfo.IPAddresses) - if err = client.AddDefaultRoutes(client.vlanEthName); err != nil { + if err = client.AddDefaultRoutes(client.vlanIfName); err != nil { return errors.Wrap(err, "failed vnet ns add default/gateway routes (idempotent)") } - if err = client.AddDefaultArp(client.vlanEthName, azureMac); err != nil { + if err = client.AddDefaultArp(client.vlanIfName, azureMac); err != nil { return errors.Wrap(err, "failed vnet ns add default arp entry (idempotent)") } if err = addRoutes(client.netlink, client.netioshim, client.vnetVethName, routeInfoList); err != nil { diff --git a/network/native_endpointclient_linux_test.go b/network/native_endpointclient_linux_test.go index 8bb794b7d3..ab81ca6041 100644 --- a/network/native_endpointclient_linux_test.go +++ b/network/native_endpointclient_linux_test.go @@ -84,8 +84,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints create vnet ns failure", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -110,8 +110,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints with existing vnet ns", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -133,8 +133,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints netlink fail", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -157,8 +157,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints get interface fail for primary interface (eth0)", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -183,8 +183,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints get interface fail for getting container veth", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -207,8 +207,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints NetNS Get fail", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -229,8 +229,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints NetNS Set fail", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -280,8 +280,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints get vnet veth mac address", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -296,8 +296,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints fail check vlan veth exists", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -313,8 +313,8 @@ func TestNativeAddEndpoints(t *testing.T) { { name: "Add endpoints fail check vnet veth exists", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -368,8 +368,8 @@ func TestNativeDeleteEndpoints(t *testing.T) { { name: "Delete endpoint delete vnet ns", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -392,8 +392,8 @@ func TestNativeDeleteEndpoints(t *testing.T) { { name: "Delete endpoint do not delete vnet ns it is still in use", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -418,8 +418,8 @@ func TestNativeDeleteEndpoints(t *testing.T) { { name: "Delete endpoint fail to delete namespace", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -474,8 +474,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { { name: "Configure interface and routes good path for container", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -498,8 +498,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { { name: "Configure interface and routes multiple IPs", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -530,8 +530,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { { name: "Configure interface and routes assign ip fail", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -555,8 +555,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { { name: "Configure interface and routes container 2nd default route added fail", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -601,8 +601,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { { name: "Configure interface and routes good path for vnet", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1", @@ -626,8 +626,8 @@ func TestNativeConfigureContainerInterfacesAndRoutes(t *testing.T) { // fail route that tells which device container ip is on for vnet name: "Configure interface and routes fail final routes for vnet", client: &NativeEndpointClient{ - eth0VethName: "eth0", - vlanEthName: "eth0.1", + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", vnetVethName: "A1veth0", containerVethName: "B1veth0", vnetNSName: "az_ns_1",