diff --git a/network/endpoint_snatroute_linux.go b/network/endpoint_snatroute_linux.go index 6b659327c3..3f45a7f16e 100644 --- a/network/endpoint_snatroute_linux.go +++ b/network/endpoint_snatroute_linux.go @@ -37,7 +37,7 @@ func AddSnatEndpointRules(snatClient *snat.Client, hostToNC, ncToHost bool, nl n return errors.Wrap(err, "failed to block ip addresses on snat bridge") } nuc := networkutils.NewNetworkUtils(nl, plc) - if err := nuc.EnableIPForwarding(snat.SnatBridgeName); err != nil { + if err := nuc.EnableIPForwarding(); err != nil { return errors.Wrap(err, "failed to enable ip forwarding") } diff --git a/network/networkutils/networkutils_linux.go b/network/networkutils/networkutils_linux.go index c6b94ddb2e..4d7473b0b5 100644 --- a/network/networkutils/networkutils_linux.go +++ b/network/networkutils/networkutils_linux.go @@ -200,8 +200,8 @@ func BlockIPAddresses(bridgeName, action string) error { return nil } -// This fucntion enables ip forwarding in VM and allow forwarding packets from the interface -func (nu NetworkUtils) EnableIPForwarding(ifName string) error { +// This function enables ip forwarding in VM and allow forwarding packets from the interface +func (nu NetworkUtils) EnableIPForwarding() error { // Enable ip forwading on linux vm. // sysctl -w net.ipv4.ip_forward=1 cmd := fmt.Sprint(enableIPForwardCmd) diff --git a/network/transparent_vlan_endpointclient_linux.go b/network/transparent_vlan_endpointclient_linux.go index 5532db20d1..56ae8517c3 100644 --- a/network/transparent_vlan_endpointclient_linux.go +++ b/network/transparent_vlan_endpointclient_linux.go @@ -26,14 +26,11 @@ const ( tunnelingTable = 2 // Packets not entering on the vlan interface go to this routing table tunnelingMark = 333 // The packets that are to tunnel will be marked with this number DisableRPFilterCmd = "sysctl -w net.ipv4.conf.all.rp_filter=0" // Command to disable the rp filter for tunneling + numRetries = 5 + sleepInMs = 100 ) -var errNamespaceCreation = fmt.Errorf("Network namespace creation error") - -var ( - numRetries = 5 - sleepInMs = 100 -) +var errNamespaceCreation = fmt.Errorf("network namespace creation error") type netnsClient interface { Get() (fileDescriptor int, err error) @@ -113,8 +110,10 @@ func NewTransparentVlanEndpointClient( // Adds interfaces to the vnet (created if not existing) and vm namespace func (client *TransparentVlanEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { // VM Namespace - err := client.PopulateVM(epInfo) - if err != nil { + if err := client.ensureCleanPopulateVM(); err != nil { + return errors.Wrap(err, "failed to ensure both network namespace and vlan veth were present or both absent") + } + if err := client.PopulateVM(epInfo); err != nil { return err } if err := client.AddSnatEndpoint(); err != nil { @@ -126,32 +125,80 @@ func (client *TransparentVlanEndpointClient) AddEndpoints(epInfo *EndpointInfo) }) } -func (client *TransparentVlanEndpointClient) createNetworkNamespace(vmNS, numRetries int) error { - var isNamespaceUnique bool - - for i := 0; i < numRetries; i++ { - vnetNS, err := client.netnsClient.NewNamed(client.vnetNSName) - if err != nil { - return errors.Wrap(err, "failed to create vnet ns") - } - logger.Info("Vnet Namespace created", zap.String("vnetNS", client.netnsClient.NamespaceUniqueID(vnetNS))) - if !client.netnsClient.IsNamespaceEqual(vnetNS, vmNS) { - client.vnetNSFileDescriptor = vnetNS - isNamespaceUnique = true - break +// Called from AddEndpoints, Namespace: VM and Vnet +func (client *TransparentVlanEndpointClient) ensureCleanPopulateVM() error { + // Clean up vlan interface in the VM namespace and ensure the network namespace (if it exists) has a vlan interface + logger.Info("Checking if NS and vlan veth exists...") + var existingErr error + client.vnetNSFileDescriptor, existingErr = client.netnsClient.GetFromName(client.vnetNSName) + if existingErr == nil { + // The namespace exists + vlanIfErr := ExecuteInNS(client.nsClient, client.vnetNSName, func() error { + // Ensure the vlan interface exists in the namespace + _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanIfName) + return errors.Wrap(err, "failed to get vlan interface in namespace") + }) + if vlanIfErr != nil { + // Assume any error is the vlan interface not found + logger.Info("vlan interface doesn't exist even though network namespace exists, deleting network namespace...", zap.String("message", vlanIfErr.Error())) + delErr := client.netnsClient.DeleteNamed(client.vnetNSName) + if delErr != nil { + return errors.Wrap(delErr, "failed to cleanup/delete ns after noticing vlan veth does not exist") + } } - logger.Info("Vnet Namespace is the same as VM namespace. Deleting and retrying...") - delErr := client.netnsClient.DeleteNamed(client.vnetNSName) - if delErr != nil { - logger.Error("failed to cleanup/delete ns after failing to create vlan veth", zap.Any("error:", delErr.Error())) + } + // Delete the vlan interface in the VM namespace if it exists + _, vlanIfInVMErr := client.netioshim.GetNetworkInterfaceByName(client.vlanIfName) + if vlanIfInVMErr == nil { + // The vlan interface exists in the VM ns because it failed to move into the network ns previously and needs to be cleaned up + logger.Info("vlan interface exists on the VM namespace, deleting", zap.String("vlanIfName", client.vlanIfName)) + if delErr := client.netlink.DeleteLink(client.vlanIfName); delErr != nil { + return errors.Wrap(delErr, "failed to clean up vlan interface") } - time.Sleep(time.Duration(sleepInMs) * time.Millisecond) } + return nil +} + +// Called from PopulateVM, Namespace: VM +func (client *TransparentVlanEndpointClient) createNetworkNamespace(vmNS int) error { + // If this call succeeds, the namespace is set to the new namespace + vnetNS, err := client.netnsClient.NewNamed(client.vnetNSName) + if err != nil { + return errors.Wrap(err, "failed to create vnet ns") + } + logger.Info("Vnet Namespace created", zap.String("vnetNS", client.netnsClient.NamespaceUniqueID(vnetNS)), zap.String("vmNS", client.netnsClient.NamespaceUniqueID(vmNS))) + if !client.netnsClient.IsNamespaceEqual(vnetNS, vmNS) { + client.vnetNSFileDescriptor = vnetNS + return nil + } + // the vnet and vm namespace are the same by this point + logger.Info("Vnet Namespace is the same as VM namespace. Deleting...") + delErr := client.netnsClient.DeleteNamed(client.vnetNSName) + if delErr != nil { + logger.Error("failed to cleanup/delete ns after noticing vnet ns is the same as vm ns", zap.Any("error:", delErr.Error())) + } + return errors.Wrap(errNamespaceCreation, "vnet and vm namespace are the same") +} - if !isNamespaceUnique { - return errors.Wrap(errNamespaceCreation, "vnet and vm namespace are same") +// Called from PopulateVM, Namespace: VM and namespace represented by fd +func (client *TransparentVlanEndpointClient) setLinkNetNSAndConfirm(name string, fd uintptr) error { + logger.Info("Move link to NS", zap.String("ifName", name), zap.Any("NSFileDescriptor", fd)) + err := client.netlink.SetLinkNetNs(name, fd) + if err != nil { + return errors.Wrapf(err, "failed to set %v inside namespace %v", name, fd) } + // confirm veth was moved successfully + err = RunWithRetries(func() error { + // retry checking in the namespace if the interface is not detected + return ExecuteInNS(client.nsClient, client.vnetNSName, func() error { + _, ifDetectedErr := client.netioshim.GetNetworkInterfaceByName(client.vlanIfName) + return errors.Wrap(ifDetectedErr, "failed to get vlan veth in namespace") + }) + }, numRetries, sleepInMs) + if err != nil { + return errors.Wrapf(err, "failed to detect %v inside namespace %v", name, fd) + } return nil } @@ -169,10 +216,13 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er // This will also (we assume) mean the vlan veth does not exist if existingErr != nil { // We assume the only possible error is that the namespace doesn't exist - logger.Info("No existing NS detected. Creating the vnet namespace and switching to it") + logger.Info("No existing NS detected. Creating the vnet namespace and switching to it", zap.String("message", existingErr.Error())) - if err = client.createNetworkNamespace(vmNS, numRetries); err != nil { - return errors.Wrap(err, "") + err = RunWithRetries(func() error { + return client.createNetworkNamespace(vmNS) + }, numRetries, sleepInMs) + if err != nil { + return errors.Wrap(err, "failed to create network namespace") } deleteNSIfNotNilErr := client.netnsClient.Set(vmNS) @@ -214,6 +264,8 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er // Any failure to add the link should error (auto delete NS) return errors.Wrap(deleteNSIfNotNilErr, "failed to create vlan vnet link after making new ns") } + // Prevent accidentally deleting NS and vlan interface since we ignore this error + deleteNSIfNotNilErr = nil } defer func() { if deleteNSIfNotNilErr != nil { @@ -225,12 +277,11 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er }() // sometimes there is slight delay in interface creation. check if it exists - for i := 0; i < numRetries; i++ { - if _, err = client.netioshim.GetNetworkInterfaceByName(client.vlanIfName); err == nil { - break - } - time.Sleep(time.Duration(sleepInMs) * time.Millisecond) - } + err = RunWithRetries(func() error { + _, err = client.netioshim.GetNetworkInterfaceByName(client.vlanIfName) + return errors.Wrap(err, "failed to get vlan veth") + }, numRetries, sleepInMs) + if err != nil { deleteNSIfNotNilErr = errors.Wrapf(err, "failed to get vlan veth interface:%s", client.vlanIfName) return deleteNSIfNotNilErr @@ -242,12 +293,13 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er } // vlan veth was created successfully, so move the vlan veth you created logger.Info("Move vlan link to vnet NS", zap.String("vlanIfName", client.vlanIfName), zap.Any("vnetNSFileDescriptor", uintptr(client.vnetNSFileDescriptor))) - deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) + deleteNSIfNotNilErr = client.setLinkNetNSAndConfirm(client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) if deleteNSIfNotNilErr != nil { - return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") + return errors.Wrap(deleteNSIfNotNilErr, "failed to move or detect vlan veth inside vnet namespace") } + logger.Info("Moving vlan veth into namespace confirmed") } else { - logger.Info("Existing NS detected. Assuming exists too", zap.String("vnetNSName", client.vnetNSName), zap.String("vlanIfName", client.vlanIfName)) + logger.Info("Existing NS detected. vlan interface should exist or namespace would've been deleted.", zap.String("vnetNSName", client.vnetNSName), zap.String("vlanIfName", client.vlanIfName)) } // Get the default constant host veth mac @@ -260,6 +312,27 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName, mac); err != nil { return errors.Wrap(err, "failed to create veth pair") } + + // Ensure vnet veth is created, as there may be a slight delay + err = RunWithRetries(func() error { + _, getErr := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) + return errors.Wrap(getErr, "failed to get vnet veth") + }, numRetries, sleepInMs) + if err != nil { + return errors.Wrap(err, "vnet veth does not exist") + } + + // Ensure container veth is created, as there may be a slight delay + var containerIf *net.Interface + err = RunWithRetries(func() error { + var getErr error + containerIf, getErr = client.netioshim.GetNetworkInterfaceByName(client.containerVethName) + return errors.Wrap(getErr, "failed to get container veth") + }, numRetries, sleepInMs) + if err != nil { + return errors.Wrap(err, "container veth does not exist") + } + // Disable RA for veth pair, and delete if any failure if err = client.netUtilsClient.DisableRAForInterface(client.vnetVethName); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { @@ -274,16 +347,11 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er return errors.Wrap(err, "failed to disable RA on container veth, deleting") } - if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { + if err = client.setLinkNetNSAndConfirm(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { logger.Error("Deleting vnet veth failed on addendpoint failure with", zap.Error(delErr)) } - return errors.Wrap(err, "failed to move vnetVethName into vnet ns, deleting") - } - - containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) - if err != nil { - return errors.Wrap(err, "container veth does not exist") + return errors.Wrap(err, "failed to move or detect vnetVethName in vnet ns, deleting") } client.containerMac = containerIf.HardwareAddr return nil @@ -492,7 +560,7 @@ func (client *TransparentVlanEndpointClient) GetVnetRoutes(ipAddresses []net.IPN // Helper that creates routing rules for the current NS which direct packets // to the virtual gateway ip on linkToName device interface -// Route 1: 169.254.1.1 dev +// Route 1: 169.254.2.1 dev // Route 2: default via 169.254.2.1 dev func (client *TransparentVlanEndpointClient) addDefaultRoutes(linkToName string, table int) error { // Add route for virtualgwip (ip route add 169.254.2.1/32 dev eth0) @@ -637,3 +705,16 @@ func ExecuteInNS(nsc NamespaceClientInterface, nsName string, f func() error) er }() return f() } + +func RunWithRetries(f func() error, maxRuns, sleepMs int) error { + var err error + for i := 0; i < maxRuns; i++ { + err = f() + if err == nil { + break + } + logger.Info("Retrying after delay...", zap.String("error", err.Error()), zap.Int("retry", i), zap.Int("sleepMs", sleepMs)) + time.Sleep(time.Duration(sleepMs) * time.Millisecond) + } + return err +} diff --git a/network/transparent_vlan_endpointclient_linux_test.go b/network/transparent_vlan_endpointclient_linux_test.go index 9600236324..be64142bc5 100644 --- a/network/transparent_vlan_endpointclient_linux_test.go +++ b/network/transparent_vlan_endpointclient_linux_test.go @@ -16,6 +16,8 @@ import ( ) var errNetnsMock = errors.New("mock netns error") +var errMockNetIOFail = errors.New("netio fail") +var errMockNetIONoIfFail = &net.OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: errors.New("no such network interface")} func newNetnsErrorMock(errStr string) error { return errors.Wrap(errNetnsMock, errStr) @@ -77,6 +79,42 @@ func defaultDeleteNamed(name string) error { return nil } +// This mock netioshim provides more flexbility in when it errors compared to the one in the netio package +type mockNetIO struct { + existingInterfaces map[string]bool + err error +} + +func (ns *mockNetIO) GetNetworkInterfaceByName(name string) (*net.Interface, error) { + if ns.existingInterfaces[name] { + hwAddr, _ := net.ParseMAC("ab:cd:ef:12:34:56") + return &net.Interface{ + //nolint:gomnd // Dummy MTU + MTU: 1000, + Name: name, + HardwareAddr: hwAddr, + //nolint:gomnd // Dummy interface index + Index: 2, + }, nil + } + return nil, errors.Wrap(ns.err, name) +} + +func (ns *mockNetIO) GetNetworkInterfaceAddrs(_ *net.Interface) ([]net.Addr, error) { + return []net.Addr{}, nil +} + +func (ns *mockNetIO) GetNetworkInterfaceByMac(mac net.HardwareAddr) (*net.Interface, error) { + return &net.Interface{ + //nolint:gomnd // Dummy MTU + MTU: 1000, + Name: "eth1", + HardwareAddr: mac, + //nolint:gomnd // Dummy interface index + Index: 2, + }, nil +} + func TestTransparentVlanAddEndpoints(t *testing.T) { nl := netlink.NewMockNetlink(false, "") plc := platform.NewMockExecClient(false) @@ -87,6 +125,180 @@ func TestTransparentVlanAddEndpoints(t *testing.T) { epInfo *EndpointInfo wantErr bool wantErrMsg string + }{ + // Set the link network namespace and confirm that it was moved inside + { + name: "Set link netns good path", + client: &TransparentVlanEndpointClient{ + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), + }, + epInfo: &EndpointInfo{}, + wantErr: false, + }, + { + name: "Set link netns fail to set", + client: &TransparentVlanEndpointClient{ + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "failed to set eth0.1", + }, + { + name: "Set link netns fail to detect", + client: &TransparentVlanEndpointClient{ + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: &mockNetIO{ + existingInterfaces: map[string]bool{}, + err: errMockNetIOFail, + }, + nsClient: NewMockNamespaceClient(), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "failed to detect eth0.1", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.setLinkNetNSAndConfirm(tt.client.vlanIfName, 1) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } + + tests = []struct { + name string + client *TransparentVlanEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + // Ensuring vnet namespace and vlan both exist or are both absent before populating the vm + { + name: "Ensure clean populate VM neither vnet ns nor vlan if exists", + client: &TransparentVlanEndpointClient{ + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: func(name string) (fileDescriptor int, err error) { + return 0, newNetnsErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), + }, + epInfo: &EndpointInfo{}, + wantErr: false, + }, + { + name: "Ensure clean populate VM vnet ns exists vlan does not exist", + client: &TransparentVlanEndpointClient{ + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: defaultGetFromName, + deleteNamed: func(name string) (err error) { + return newNetnsErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: &mockNetIO{ + existingInterfaces: map[string]bool{}, + err: errMockNetIONoIfFail, + }, + nsClient: NewMockNamespaceClient(), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "failed to cleanup/delete ns after noticing vlan veth does not exist: netns failure: " + errNetnsMock.Error(), + }, + { + name: "Ensure clean populate VM cleanup straggling vlan if in vm ns", + client: &TransparentVlanEndpointClient{ + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: func(name string) (fileDescriptor int, err error) { + return 0, newNetnsErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "failed to clean up vlan interface", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.ensureCleanPopulateVM() + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } + tests = []struct { + name string + client *TransparentVlanEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string }{ // Populating VM with data and creating interfaces/links { @@ -134,6 +346,7 @@ func TestTransparentVlanAddEndpoints(t *testing.T) { plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), }, epInfo: &EndpointInfo{}, wantErr: false, @@ -160,7 +373,7 @@ func TestTransparentVlanAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to move vnetVethName into vnet ns, deleting: " + netlink.ErrorMockNetlink.Error() + " : netlink fail", + wantErrMsg: "failed to move or detect vnetVethName in vnet ns, deleting: failed to set A1veth0 inside namespace 1: " + netlink.ErrorMockNetlink.Error() + " : netlink fail", }, { name: "Add endpoints get interface fail for primary interface (eth0)", @@ -206,11 +419,17 @@ func TestTransparentVlanAddEndpoints(t *testing.T) { netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 1), + netioshim: &mockNetIO{ + existingInterfaces: map[string]bool{ + "A1veth0": true, + }, + err: errMockNetIOFail, + }, + nsClient: NewMockNamespaceClient(), }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "container veth does not exist: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + wantErrMsg: "container veth does not exist: failed to get container veth: B1veth0: " + errMockNetIOFail.Error() + "", }, { name: "Add endpoints NetNS Get fail", @@ -235,7 +454,7 @@ func TestTransparentVlanAddEndpoints(t *testing.T) { wantErrMsg: "failed to get vm ns handle: netns failure: " + errNetnsMock.Error(), }, { - name: "Add endpoints NetNS Set fail", + name: "Add endpoints no vnet ns NetNS Set fail", client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", @@ -689,3 +908,58 @@ func TestTransparentVlanConfigureContainerInterfacesAndRoutes(t *testing.T) { }) } } + +func createFunctionWithFailurePattern(errorPattern []error) func() error { + s := 0 + return func() error { + if s >= len(errorPattern) { + return nil + } + result := errorPattern[s] + s++ + return result + } +} + +func TestRunWithRetries(t *testing.T) { + errMock := errors.New("mock error") + runs := 4 + + tests := []struct { + name string + wantErr bool + f func() error + }{ + { + name: "Succeed on first try", + f: createFunctionWithFailurePattern([]error{}), + wantErr: false, + }, + { + name: "Succeed on first try do not check again", + f: createFunctionWithFailurePattern([]error{nil, errMock, errMock, errMock}), + wantErr: false, + }, + { + name: "Succeed on last try", + f: createFunctionWithFailurePattern([]error{errMock, errMock, errMock, nil, errMock}), + wantErr: false, + }, + { + name: "Fail after too many attempts", + f: createFunctionWithFailurePattern([]error{errMock, errMock, errMock, errMock, nil, nil}), + wantErr: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := RunWithRetries(tt.f, runs, 100) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +}