From a580917580d50bd07b6f45fc376c3a239505982e Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 8 Sep 2023 17:35:49 +0000 Subject: [PATCH 01/64] feat: update contracts to support swift 2 --- cni/network/invoker.go | 12 ++++++++++-- cni/network/invoker_cns.go | 4 ++++ network/endpoint.go | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index 0d111933f8..38386cfc4e 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -27,8 +27,16 @@ type IPAMAddConfig struct { } type IPAMAddResult struct { - ipv4Result *cniTypesCurr.Result - ipv6Result *cniTypesCurr.Result + defaultCniResult CNIResult + cniResults []CNIResult ncResponse *cns.GetNetworkContainerResponse hostSubnetPrefix net.IPNet + ipv6Enabled bool +} + +type CNIResult struct { + ipResult *cniTypesCurr.Result + addressType string + macAddress net.HardwareAddr + isDefaultInterface bool } diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index ad60a2fb0a..73c99b8b9f 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -44,6 +44,10 @@ type IPResultInfo struct { hostSubnet string hostPrimaryIP string hostGateway string + addressType string + macAddress string + isDefaultInterface bool + routes []cns.Route } func NewCNSInvoker(podName, namespace string, cnsClient cnsclient, executionMode util.ExecutionMode, ipamMode util.IpamMode) *CNSIPAMInvoker { diff --git a/network/endpoint.go b/network/endpoint.go index a8fae33776..4e20b2d7f0 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -52,6 +52,7 @@ type endpoint struct { PODNameSpace string `json:",omitempty"` InfraVnetAddressSpace string `json:",omitempty"` NetNs string `json:",omitempty"` + SecondaryInterfaces map[string]*InterfaceInfo } // EndpointInfo contains read-only information about an endpoint. @@ -86,6 +87,8 @@ type EndpointInfo struct { VnetCidrs string ServiceCidrs string NATInfo []policy.NATInfo + AddressType string + IsDefaultInterface bool } // RouteInfo contains information about an IP route. @@ -100,6 +103,17 @@ type RouteInfo struct { Table int } +// InterfaceInfo contains information for secondary interfaces +type InterfaceInfo struct { + Name string + MacAddress net.HardwareAddr + IPAddress []net.IPNet + Gateways []net.IP + Routes []RouteInfo + AddressType string + IsDefaultInterface bool +} + type apipaClient interface { DeleteHostNCApipaEndpoint(ctx context.Context, networkContainerID string) error CreateHostNCApipaEndpoint(ctx context.Context, networkContainerID string) (string, error) From 22badf9a1e6e49cfdebdcb9af46fa12fcadfb26b Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 18 Sep 2023 20:01:27 +0000 Subject: [PATCH 02/64] add comments --- cni/network/invoker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index 38386cfc4e..46be4114bc 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -27,6 +27,7 @@ type IPAMAddConfig struct { } type IPAMAddResult struct { + // Splitting defaultCniResult from (secondary) cniResults so we don't need to loop for default CNI result every time defaultCniResult CNIResult cniResults []CNIResult ncResponse *cns.GetNetworkContainerResponse From a0afdfbc2518a176dd8150cfb3e44289bc2eb6e6 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 22 Sep 2023 19:24:53 +0000 Subject: [PATCH 03/64] rename AddressType to NICType --- cni/network/invoker.go | 2 +- cni/network/invoker_cns.go | 2 +- network/endpoint.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index 46be4114bc..92742cb706 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -37,7 +37,7 @@ type IPAMAddResult struct { type CNIResult struct { ipResult *cniTypesCurr.Result - addressType string + nicType string macAddress net.HardwareAddr isDefaultInterface bool } diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 73c99b8b9f..0ebe55c891 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -44,7 +44,7 @@ type IPResultInfo struct { hostSubnet string hostPrimaryIP string hostGateway string - addressType string + nicType string macAddress string isDefaultInterface bool routes []cns.Route diff --git a/network/endpoint.go b/network/endpoint.go index 4e20b2d7f0..da81cfc557 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -87,7 +87,7 @@ type EndpointInfo struct { VnetCidrs string ServiceCidrs string NATInfo []policy.NATInfo - AddressType string + NICType string IsDefaultInterface bool } @@ -110,7 +110,7 @@ type InterfaceInfo struct { IPAddress []net.IPNet Gateways []net.IP Routes []RouteInfo - AddressType string + NICType string IsDefaultInterface bool } From f8baa47a0169fea19d32f0e9de153baa2c3ee445 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 25 Sep 2023 19:25:42 +0000 Subject: [PATCH 04/64] update contract names and comments --- cni/network/invoker.go | 17 +++++++++-------- cni/network/invoker_cns.go | 2 +- network/endpoint.go | 16 ++++++++-------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index 92742cb706..78f87f14a6 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -27,17 +27,18 @@ type IPAMAddConfig struct { } type IPAMAddResult struct { - // Splitting defaultCniResult from (secondary) cniResults so we don't need to loop for default CNI result every time - defaultCniResult CNIResult - cniResults []CNIResult + // Splitting defaultInterfaceInfo from secondaryInterfaceInfo so we don't need to loop for default CNI result every time + defaultInterfaceInfo InterfaceInfo + secondaryInterfaceInfo []InterfaceInfo + // ncResponse is used for Swift 1.0 multitenancy ncResponse *cns.GetNetworkContainerResponse hostSubnetPrefix net.IPNet ipv6Enabled bool } -type CNIResult struct { - ipResult *cniTypesCurr.Result - nicType string - macAddress net.HardwareAddr - isDefaultInterface bool +type InterfaceInfo struct { + ipResult *cniTypesCurr.Result + nicType string + macAddress net.HardwareAddr + skipDefaultRoutes bool } diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 0ebe55c891..4df741cade 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -46,7 +46,7 @@ type IPResultInfo struct { hostGateway string nicType string macAddress string - isDefaultInterface bool + skipDefaultRoutes bool routes []cns.Route } diff --git a/network/endpoint.go b/network/endpoint.go index da81cfc557..95ffe33d8d 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -88,7 +88,7 @@ type EndpointInfo struct { ServiceCidrs string NATInfo []policy.NATInfo NICType string - IsDefaultInterface bool + SkipDefaultRoutes bool } // RouteInfo contains information about an IP route. @@ -105,13 +105,13 @@ type RouteInfo struct { // InterfaceInfo contains information for secondary interfaces type InterfaceInfo struct { - Name string - MacAddress net.HardwareAddr - IPAddress []net.IPNet - Gateways []net.IP - Routes []RouteInfo - NICType string - IsDefaultInterface bool + Name string + MacAddress net.HardwareAddr + IPAddress []net.IPNet + Gateways []net.IP + Routes []RouteInfo + NICType string + SkipDefaultRoutes bool } type apipaClient interface { From 6fdc198dce316f64f140110cfe36899e99bb4aba Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Tue, 3 Oct 2023 16:14:05 +0000 Subject: [PATCH 05/64] address comments --- network/endpoint.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/endpoint.go b/network/endpoint.go index 95ffe33d8d..25d5c00871 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -52,7 +52,8 @@ type endpoint struct { PODNameSpace string `json:",omitempty"` InfraVnetAddressSpace string `json:",omitempty"` NetNs string `json:",omitempty"` - SecondaryInterfaces map[string]*InterfaceInfo + // SecondaryInterfaces is a map of interface name to InterfaceInfo + SecondaryInterfaces map[string]*InterfaceInfo } // EndpointInfo contains read-only information about an endpoint. @@ -108,7 +109,6 @@ type InterfaceInfo struct { Name string MacAddress net.HardwareAddr IPAddress []net.IPNet - Gateways []net.IP Routes []RouteInfo NICType string SkipDefaultRoutes bool From 5ad7eeaf36ea38253a370c4204d6de2067a95616 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 8 Sep 2023 18:02:44 +0000 Subject: [PATCH 06/64] feat: update invokers to support swift 2 --- cni/network/invoker_azure.go | 26 ++-- cni/network/invoker_azure_test.go | 53 +++----- cni/network/invoker_cns.go | 204 ++++++++++++++++++++---------- cni/network/invoker_cns_test.go | 166 ++++++++++++++++++------ cni/network/invoker_mock.go | 14 +- cni/network/multitenancy.go | 3 +- cni/network/multitenancy_mock.go | 3 +- 7 files changed, 304 insertions(+), 165 deletions(-) diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index 9b8303c39a..f8995b53a3 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cni/log" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/ipam" "github.com/Azure/azure-container-networking/network" @@ -46,10 +47,7 @@ func NewAzureIpamInvoker(plugin *NetPlugin, nwInfo *network.NetworkInfo) *AzureI } func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, error) { - var ( - addResult = IPAMAddResult{} - err error - ) + addResult := IPAMAddResult{} if addConfig.nwCfg == nil { return addResult, invoker.plugin.Errorf("nil nwCfg passed to CNI ADD, stack: %+v", string(debug.Stack())) @@ -60,8 +58,7 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er } // Call into IPAM plugin to allocate an address pool for the network. - addResult.ipv4Result, err = invoker.plugin.DelegateAdd(addConfig.nwCfg.IPAM.Type, addConfig.nwCfg) - + ipv4Result, err := invoker.plugin.DelegateAdd(addConfig.nwCfg.IPAM.Type, addConfig.nwCfg) if err != nil && strings.Contains(err.Error(), ipam.ErrNoAvailableAddressPools.Error()) { invoker.deleteIpamState() logger.Info("Retry pool allocation after deleting IPAM state") @@ -72,11 +69,15 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er err = invoker.plugin.Errorf("Failed to allocate pool: %v", err) return addResult, err } + addResult.defaultCniResult = CNIResult{ipResult: ipv4Result, addressType: cns.Default, isDefaultInterface: true} + if len(ipv4Result.IPs) > 0 { + addResult.hostSubnetPrefix = ipv4Result.IPs[0].Address + } defer func() { if err != nil { - if len(addResult.ipv4Result.IPs) > 0 { - if er := invoker.Delete(&addResult.ipv4Result.IPs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { + if len(addResult.defaultCniResult.ipResult.IPs) > 0 { + if er := invoker.Delete(&addResult.defaultCniResult.ipResult.IPs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { err = invoker.plugin.Errorf("Failed to clean up IP's during Delete with error %v, after Add failed with error %w", er, err) } } else { @@ -95,14 +96,17 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er nwCfg6.IPAM.Subnet = invoker.nwInfo.Subnets[1].Prefix.String() } - addResult.ipv6Result, err = invoker.plugin.DelegateAdd(nwCfg6.IPAM.Type, &nwCfg6) + var ipv6Result *cniTypesCurr.Result + ipv6Result, err = invoker.plugin.DelegateAdd(nwCfg6.IPAM.Type, &nwCfg6) if err != nil { err = invoker.plugin.Errorf("Failed to allocate v6 pool: %v", err) + } else { + addResult.defaultCniResult.ipResult.IPs = append(addResult.defaultCniResult.ipResult.IPs, ipv6Result.IPs...) + addResult.defaultCniResult.ipResult.Routes = append(addResult.defaultCniResult.ipResult.Routes, ipv6Result.Routes...) + addResult.ipv6Enabled = true } } - addResult.hostSubnetPrefix = addResult.ipv4Result.IPs[0].Address - return addResult, err } diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index ad9e76b76d..51ce194c8b 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -22,12 +22,10 @@ type mockDelegatePlugin struct { } type add struct { - resultsIPv4Index int - resultsIPv4 [](*cniTypesCurr.Result) - resultsIPv6Index int - resultsIPv6 [](*cniTypesCurr.Result) - errv4 error - errv6 error + resultsIPv4 *cniTypesCurr.Result + resultsIPv6 *cniTypesCurr.Result + errv4 error + errv6 error } func (d *add) DelegateAdd(pluginName string, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { @@ -35,23 +33,13 @@ func (d *add) DelegateAdd(pluginName string, nwCfg *cni.NetworkConfig) (*cniType if d.errv6 != nil { return nil, d.errv6 } - if d.resultsIPv6 == nil || d.resultsIPv6Index-1 > len(d.resultsIPv6) { - return nil, errors.New("no more ipv6 results in mock available") //nolint:goerr113 - } - res := d.resultsIPv6[d.resultsIPv6Index] - d.resultsIPv6Index++ - return res, nil + return d.resultsIPv6, nil } if d.errv4 != nil { return nil, d.errv4 } - if d.resultsIPv4 == nil || d.resultsIPv4Index-1 > len(d.resultsIPv4) { - return nil, errors.New("no more ipv4 results in mock available") //nolint:goerr113 - } - res := d.resultsIPv4[d.resultsIPv4Index] - d.resultsIPv4Index++ - return res, nil + return d.resultsIPv4, nil } type del struct { @@ -82,15 +70,10 @@ func getCIDRNotationForAddress(ipaddresswithcidr string) *net.IPNet { return ipnet } -func getResult(ip string) []*cniTypesCurr.Result { - res := []*cniTypesCurr.Result{ - { - IPs: []*cniTypesCurr.IPConfig{ - { - Address: *getCIDRNotationForAddress(ip), - }, - }, - }, +func getResult(ips ...string) *cniTypesCurr.Result { + res := &cniTypesCurr.Result{} + for _, ip := range ips { + res.IPs = append(res.IPs, &cniTypesCurr.IPConfig{Address: *getCIDRNotationForAddress(ip)}) } return res } @@ -127,7 +110,6 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { fields fields args args want *cniTypesCurr.Result - want1 *cniTypesCurr.Result wantErr bool }{ { @@ -145,7 +127,7 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { nwCfg: &cni.NetworkConfig{}, subnetPrefix: getCIDRNotationForAddress("10.0.0.0/24"), }, - want: getResult("10.0.0.1/24")[0], + want: getResult("10.0.0.1/24"), wantErr: false, }, { @@ -165,8 +147,7 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { }, subnetPrefix: getCIDRNotationForAddress("10.0.0.0/24"), }, - want: getResult("10.0.0.1/24")[0], - want1: getResult("2001:0db8:abcd:0015::0/64")[0], + want: getResult("10.0.0.1/24", "2001:0db8:abcd:0015::0/64"), wantErr: false, }, { @@ -183,7 +164,6 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { nwCfg: &cni.NetworkConfig{}, }, want: nil, - want1: nil, wantErr: true, }, { @@ -203,8 +183,7 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { }, subnetPrefix: getCIDRNotationForAddress("10.0.0.0/24"), }, - want: getResult("10.0.0.1/24")[0], - want1: nil, + want: getResult("10.0.0.1/24"), wantErr: true, }, } @@ -226,8 +205,8 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { require.Nil(err) } - require.Exactly(tt.want, ipamAddResult.ipv4Result) - require.Exactly(tt.want1, ipamAddResult.ipv6Result) + fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ipamAddResult.defaultCniResult.ipResult) + require.Exactly(tt.want, ipamAddResult.defaultCniResult.ipResult) }) } } @@ -403,7 +382,7 @@ func TestRemoveIpamState_Add(t *testing.T) { nwCfg: &cni.NetworkConfig{}, subnetPrefix: getCIDRNotationForAddress("10.0.0.0/24"), }, - want: getResult("10.0.0.1/24")[0], + want: getResult("10.0.0.1/24"), wantErrMsg: ipam.ErrNoAvailableAddressPools.Error(), wantErr: true, }, diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 4df741cade..b7ac8e52ff 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -122,6 +122,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } addResult := IPAMAddResult{} + var isDefaultInterfaceSet bool for i := 0; i < len(response.PodIPInfo); i++ { info := IPResultInfo{ @@ -132,97 +133,166 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro hostSubnet: response.PodIPInfo[i].HostPrimaryIPInfo.Subnet, hostPrimaryIP: response.PodIPInfo[i].HostPrimaryIPInfo.PrimaryIP, hostGateway: response.PodIPInfo[i].HostPrimaryIPInfo.Gateway, - } - - // set the NC Primary IP in options - // SNATIPKey is not set for ipv6 - if net.ParseIP(info.ncPrimaryIP).To4() != nil { - addConfig.options[network.SNATIPKey] = info.ncPrimaryIP + addressType: response.PodIPInfo[i].AddressType, + macAddress: response.PodIPInfo[i].MacAddress, + isDefaultInterface: response.PodIPInfo[i].IsDefaultInterface, + routes: response.PodIPInfo[i].Routes, } logger.Info("Received info for pod", - zap.Any("ipv4info", info), + zap.Any("ipinfo", info), zap.Any("podInfo", podInfo)) - ip, ncIPNet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix)) - if ip == nil { - return IPAMAddResult{}, errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") - } - ncgw := net.ParseIP(info.ncGatewayIPAddress) - if ncgw == nil { - // TODO: Remove v4overlay and dualstackoverlay options, after 'overlay' rolls out in AKS-RP - if (invoker.ipamMode != util.V4Overlay) && (invoker.ipamMode != util.DualStackOverlay) && (invoker.ipamMode != util.Overlay) { - return IPAMAddResult{}, errors.Wrap(errInvalidArgs, "%w: Gateway address "+info.ncGatewayIPAddress+" from response is invalid") + switch info.addressType { + case cns.Secondary: + ip, ipnet, err := response.PodIPInfo[i].PodIPConfig.GetIPNet() + if ip == nil { + return IPAMAddResult{}, errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") } - if net.ParseIP(info.podIPAddress).To4() != nil { //nolint:gocritic - ncgw, err = getOverlayGateway(ncIPNet) - if err != nil { - return IPAMAddResult{}, err - } - } else if net.ParseIP(info.podIPAddress).To16() != nil { - ncgw = net.ParseIP(overlayGatewayV6IP) - } else { - return IPAMAddResult{}, errors.Wrap(err, "No podIPAddress is found: %w") + macAddress, err := net.ParseMAC(info.macAddress) + if err != nil { + return IPAMAddResult{}, errors.Wrap(err, "Invalid mac address") } - } - - // construct ipnet for result - resultIPnet := net.IPNet{ - IP: ip, - Mask: ncIPNet.Mask, - } - if net.ParseIP(info.podIPAddress).To4() != nil { - addResult.ipv4Result = &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { - Address: resultIPnet, - Gateway: ncgw, + isDefaultInterfaceSet = isDefaultInterfaceSet || info.isDefaultInterface + result := CNIResult{ + ipResult: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: net.IPNet{ + IP: ip, + Mask: ipnet.Mask, + }, + }, }, - }, - Routes: []*cniTypes.Route{ - { - Dst: network.Ipv4DefaultRouteDstPrefix, - GW: ncgw, + Interfaces: []*cniTypesCurr.Interface{ + { + Mac: info.macAddress, + }, }, }, + addressType: cns.Secondary, + macAddress: macAddress, + isDefaultInterface: info.isDefaultInterface, + } + + for _, route := range info.routes { + _, dst, err := net.ParseCIDR(route.IPAddress) + if err != nil { + return IPAMAddResult{}, fmt.Errorf("unable to parse destination %s: %w", route.IPAddress, err) + } + gw := net.ParseIP(route.GatewayIPAddress) + if gw == nil { + return IPAMAddResult{}, fmt.Errorf("unable to parse gateway %s: %w", route.GatewayIPAddress, err) + } + + result.ipResult.Routes = append(result.ipResult.Routes, + &cniTypes.Route{ + Dst: *dst, + GW: gw, + }) } - } else if net.ParseIP(info.podIPAddress).To16() != nil { - addResult.ipv6Result = &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { + + addResult.cniResults = append(addResult.cniResults, result) + default: + // set the NC Primary IP in options + // SNATIPKey is not set for ipv6 + if net.ParseIP(info.ncPrimaryIP).To4() != nil { + addConfig.options[network.SNATIPKey] = info.ncPrimaryIP + } + + ip, ncIPNet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix)) + if ip == nil { + return IPAMAddResult{}, errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") + } + + ncgw := net.ParseIP(info.ncGatewayIPAddress) + if ncgw == nil { + // TODO: Remove v4overlay and dualstackoverlay options, after 'overlay' rolls out in AKS-RP + if (invoker.ipamMode != util.V4Overlay) && (invoker.ipamMode != util.DualStackOverlay) && (invoker.ipamMode != util.Overlay) { + return IPAMAddResult{}, errors.Wrap(errInvalidArgs, "%w: Gateway address "+info.ncGatewayIPAddress+" from response is invalid") + } + + if net.ParseIP(info.podIPAddress).To4() != nil { //nolint:gocritic + ncgw, err = getOverlayGateway(ncIPNet) + if err != nil { + return IPAMAddResult{}, err + } + } else if net.ParseIP(info.podIPAddress).To16() != nil { + ncgw = net.ParseIP(overlayGatewayV6IP) + } else { + return IPAMAddResult{}, errors.Wrap(err, "No podIPAddress is found: %w") + } + } + + // construct ipnet for result + resultIPnet := net.IPNet{ + IP: ip, + Mask: ncIPNet.Mask, + } + + if ip := net.ParseIP(info.podIPAddress); ip != nil { + if addResult.defaultCniResult.ipResult == nil { + addResult.defaultCniResult.ipResult = &cniTypesCurr.Result{} + } + + defaultRouteDstPrefix := network.Ipv4DefaultRouteDstPrefix + if ip.To4() == nil { + defaultRouteDstPrefix = network.Ipv6DefaultRouteDstPrefix + addResult.ipv6Enabled = true + } + + addResult.defaultCniResult.ipResult.IPs = append(addResult.defaultCniResult.ipResult.IPs, + &cniTypesCurr.IPConfig{ Address: resultIPnet, Gateway: ncgw, - }, - }, - Routes: []*cniTypes.Route{ - { - Dst: network.Ipv6DefaultRouteDstPrefix, + }) + addResult.defaultCniResult.ipResult.Routes = append(addResult.defaultCniResult.ipResult.Routes, + &cniTypes.Route{ + Dst: defaultRouteDstPrefix, GW: ncgw, - }, - }, + }) + for _, route := range info.routes { + _, dst, routeErr := net.ParseCIDR(route.IPAddress) + if routeErr != nil { + return IPAMAddResult{}, fmt.Errorf("unable to parse destination %s: %w", route.IPAddress, routeErr) + } + + gw := net.ParseIP(route.GatewayIPAddress) + if gw == nil { + return IPAMAddResult{}, fmt.Errorf("unable to parse gateway %s: %w", route.GatewayIPAddress, routeErr) + } + + addResult.defaultCniResult.ipResult.Routes = append(addResult.defaultCniResult.ipResult.Routes, + &cniTypes.Route{ + Dst: *dst, + GW: gw, + }) + } } - } - // get the name of the primary IP address - _, hostIPNet, err := net.ParseCIDR(info.hostSubnet) - if err != nil { - return IPAMAddResult{}, fmt.Errorf("unable to parse hostSubnet: %w", err) - } + // get the name of the primary IP address + _, hostIPNet, err := net.ParseCIDR(info.hostSubnet) + if err != nil { + return IPAMAddResult{}, fmt.Errorf("unable to parse hostSubnet: %w", err) + } - addResult.hostSubnetPrefix = *hostIPNet + addResult.hostSubnetPrefix = *hostIPNet - // set subnet prefix for host vm - // setHostOptions will execute if IPAM mode is not v4 overlay and not dualStackOverlay mode - // TODO: Remove v4overlay and dualstackoverlay options, after 'overlay' rolls out in AKS-RP - if (invoker.ipamMode != util.V4Overlay) && (invoker.ipamMode != util.DualStackOverlay) && (invoker.ipamMode != util.Overlay) { - if err := setHostOptions(ncIPNet, addConfig.options, &info); err != nil { - return IPAMAddResult{}, err + // set subnet prefix for host vm + // setHostOptions will execute if IPAM mode is not v4 overlay and not dualStackOverlay mode + // TODO: Remove v4overlay and dualstackoverlay options, after 'overlay' rolls out in AKS-RP + if (invoker.ipamMode != util.V4Overlay) && (invoker.ipamMode != util.DualStackOverlay) && (invoker.ipamMode != util.Overlay) { + if err := setHostOptions(ncIPNet, addConfig.options, &info); err != nil { + return IPAMAddResult{}, err + } } } } + addResult.defaultCniResult.isDefaultInterface = !isDefaultInterfaceSet + return addResult, nil } diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 982f2cbacc..981e4f48f8 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -65,12 +65,12 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { } tests := []struct { - name string - fields fields - args args - wantIpv4Result *cniTypesCurr.Result - wantIpv6Result *cniTypesCurr.Result - wantErr bool + name string + fields fields + args args + wantDefaultResult *cniTypesCurr.Result + wantMultitenantResult *cniTypesCurr.Result + wantErr bool }{ { name: "Test happy CNI Overlay add in v4overlay ipamMode", @@ -126,7 +126,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { hostSubnetPrefix: getCIDRNotationForAddress("10.224.0.0/16"), options: map[string]interface{}{}, }, - wantIpv4Result: &cniTypesCurr.Result{ + wantDefaultResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { Address: *getCIDRNotationForAddress("10.240.1.242/16"), @@ -140,8 +140,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { }, }, }, - wantIpv6Result: nil, - wantErr: false, + wantErr: false, }, { name: "Test happy CNI Overlay add in dualstack overlay ipamMode", @@ -213,31 +212,117 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), options: map[string]interface{}{}, }, - wantIpv4Result: &cniTypesCurr.Result{ + wantDefaultResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { Address: *getCIDRNotationForAddress("10.0.1.10/24"), Gateway: net.ParseIP("10.0.0.1"), }, + { + Address: *getCIDRNotationForAddress("fd11:1234::1/112"), + Gateway: net.ParseIP("fe80::1234:5678:9abc"), + }, }, Routes: []*cniTypes.Route{ { Dst: network.Ipv4DefaultRouteDstPrefix, GW: net.ParseIP("10.0.0.1"), }, + { + Dst: network.Ipv6DefaultRouteDstPrefix, + GW: net.ParseIP("fe80::1234:5678:9abc"), + }, }, }, - wantIpv6Result: &cniTypesCurr.Result{ + wantErr: false, + }, + { + name: "Test happy CNI add with multitenant result", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + ipamMode: util.Overlay, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: cns.IPConfigsRequest{ + PodInterfaceID: "testcont-testifname3", + InfraContainerID: "testcontainerid3", + OrchestratorContext: marshallPodInfo(testPodInfo), + }, + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + DNSServers: nil, + GatewayIPAddress: "10.0.0.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", + }, + AddressType: cns.Default, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "20.240.1.242", + PrefixLength: 24, + }, + AddressType: cns.Secondary, + MacAddress: "12:34:56:78:9a:bc", + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid3", + Netns: "testnetns3", + IfName: "testifname3", + }, + hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), + options: map[string]interface{}{}, + }, + wantDefaultResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { - Address: *getCIDRNotationForAddress("fd11:1234::1/112"), - Gateway: net.ParseIP("fe80::1234:5678:9abc"), + Address: *getCIDRNotationForAddress("10.0.1.10/24"), + Gateway: net.ParseIP("10.0.0.1"), }, }, Routes: []*cniTypes.Route{ { - Dst: network.Ipv6DefaultRouteDstPrefix, - GW: net.ParseIP("fe80::1234:5678:9abc"), + Dst: network.Ipv4DefaultRouteDstPrefix, + GW: net.ParseIP("10.0.0.1"), + }, + }, + }, + wantMultitenantResult: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: *getCIDRNotationForAddress("20.240.1.242/24"), + }, + }, + Interfaces: []*cniTypesCurr.Interface{ + { + Mac: "12:34:56:78:9a:bc", }, }, }, @@ -262,9 +347,11 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { require.NoError(err) } - fmt.Printf("want:%+v\nrest:%+v\n", tt.wantIpv4Result, ipamAddResult.ipv4Result) - require.Equalf(tt.wantIpv4Result, ipamAddResult.ipv4Result, "incorrect ipv4 response") - require.Equalf(tt.wantIpv6Result, ipamAddResult.ipv6Result, "incorrect ipv6 response") + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.cniResults) + require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultCniResult.ipResult, "incorrect default response") + if tt.wantMultitenantResult != nil { + require.Equalf(tt.wantMultitenantResult, ipamAddResult.cniResults[0].ipResult, "incorrect multitenant response") + } }) } } @@ -285,12 +372,12 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { } tests := []struct { - name string - fields fields - args args - wantIpv4Result *cniTypesCurr.Result - wantIpv6Result *cniTypesCurr.Result - wantErr bool + name string + fields fields + args args + wantDefaultResult *cniTypesCurr.Result + wantMultitenantResult *cniTypesCurr.Result + wantErr bool }{ { name: "Test happy CNI add", @@ -342,7 +429,7 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), options: map[string]interface{}{}, }, - wantIpv4Result: &cniTypesCurr.Result{ + wantDefaultResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { Address: *getCIDRNotationForAddress("10.0.1.10/24"), @@ -356,8 +443,7 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { }, }, }, - wantIpv6Result: nil, - wantErr: false, + wantErr: false, }, { name: "Test happy CNI add for both ipv4 and ipv6", @@ -428,28 +514,22 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), options: map[string]interface{}{}, }, - wantIpv4Result: &cniTypesCurr.Result{ + wantDefaultResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { Address: *getCIDRNotationForAddress("10.0.1.10/24"), Gateway: net.ParseIP("10.0.0.1"), }, - }, - Routes: []*cniTypes.Route{ - { - Dst: network.Ipv4DefaultRouteDstPrefix, - GW: net.ParseIP("10.0.0.1"), - }, - }, - }, - wantIpv6Result: &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ { Address: *getCIDRNotationForAddress("fd11:1234::1/112"), Gateway: net.ParseIP("fe80::1234:5678:9abc"), }, }, Routes: []*cniTypes.Route{ + { + Dst: network.Ipv4DefaultRouteDstPrefix, + GW: net.ParseIP("10.0.0.1"), + }, { Dst: network.Ipv6DefaultRouteDstPrefix, GW: net.ParseIP("fe80::1234:5678:9abc"), @@ -493,9 +573,11 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { require.NoError(err) } - fmt.Printf("want:%+v\nrest:%+v\n", tt.wantIpv4Result, ipamAddResult.ipv4Result) - require.Equalf(tt.wantIpv4Result, ipamAddResult.ipv4Result, "incorrect ipv4 response") - require.Equalf(tt.wantIpv6Result, ipamAddResult.ipv6Result, "incorrect ipv6 response") + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.cniResults) + require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultCniResult.ipResult, "incorrect default response") + if tt.wantMultitenantResult != nil { + require.Equalf(tt.wantMultitenantResult, ipamAddResult.cniResults[0].ipResult, "incorrect multitenant response") + } }) } } @@ -610,7 +692,7 @@ func TestCNSIPAMInvoker_Add_UnsupportedAPI(t *testing.T) { t.Fatalf("expected an error %+v but none received", err) } require.NoError(err) - require.Equalf(tt.want, ipamAddResult.ipv4Result, "incorrect ipv4 response") + require.Equalf(tt.want, ipamAddResult.defaultCniResult.ipResult, "incorrect ipv4 response") }) } } diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index 51e04c29ce..845c883217 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -5,6 +5,7 @@ import ( "net" "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cns" "github.com/containernetworking/cni/pkg/skel" current "github.com/containernetworking/cni/pkg/types/100" ) @@ -53,9 +54,12 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ip := net.ParseIP(ipv4Str) ipnet := net.IPNet{IP: ip, Mask: net.CIDRMask(subnetBits, ipv4Bits)} gwIP := net.ParseIP("10.240.0.1") - ipConfig := ¤t.IPConfig{Address: ipnet, Gateway: gwIP} - ipamAddResult.ipv4Result = ¤t.Result{} - ipamAddResult.ipv4Result.IPs = append(ipamAddResult.ipv4Result.IPs, ipConfig) + ipamAddResult.defaultCniResult = CNIResult{ + ipResult: ¤t.Result{ + IPs: []*current.IPConfig{{Address: ipnet, Gateway: gwIP}}, + }, + addressType: cns.Default, + } invoker.ipMap[ipnet.String()] = true if invoker.v6Fail { return ipamAddResult, errV6 @@ -70,9 +74,7 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ip := net.ParseIP(ipv6Str) ipnet := net.IPNet{IP: ip, Mask: net.CIDRMask(subnetv6Bits, ipv6Bits)} gwIP := net.ParseIP("fc00::1") - ipConfig := ¤t.IPConfig{Address: ipnet, Gateway: gwIP} - ipamAddResult.ipv6Result = ¤t.Result{} - ipamAddResult.ipv6Result.IPs = append(ipamAddResult.ipv6Result.IPs, ipConfig) + ipamAddResult.defaultCniResult.ipResult.IPs = append(ipamAddResult.defaultCniResult.ipResult.IPs, ¤t.IPConfig{Address: ipnet, Gateway: gwIP}) invoker.ipMap[ipnet.String()] = true } diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 29eddc3d4b..6cf3bf31a5 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -220,7 +220,8 @@ func (m *Multitenancy) GetAllNetworkContainers( for i := 0; i < len(ncResponses); i++ { ipamResults[i].ncResponse = &ncResponses[i] ipamResults[i].hostSubnetPrefix = hostSubnetPrefixes[i] - ipamResults[i].ipv4Result = convertToCniResult(ipamResults[i].ncResponse, ifName) + ipamResults[i].defaultCniResult.ipResult = convertToCniResult(ipamResults[i].ncResponse, ifName) + ipamResults[i].defaultCniResult.addressType = cns.Default } return ipamResults, err diff --git a/cni/network/multitenancy_mock.go b/cni/network/multitenancy_mock.go index d6b826c9ba..d99fd50347 100644 --- a/cni/network/multitenancy_mock.go +++ b/cni/network/multitenancy_mock.go @@ -158,7 +158,8 @@ func (m *MockMultitenancy) GetAllNetworkContainers( for i := 0; i < len(cnsResponses); i++ { ipamResults[i].ncResponse = &cnsResponses[i] ipamResults[i].hostSubnetPrefix = ipNets[i] - ipamResults[i].ipv4Result = convertToCniResult(ipamResults[i].ncResponse, ifName) + ipamResults[i].defaultCniResult.ipResult = convertToCniResult(ipamResults[i].ncResponse, ifName) + ipamResults[i].defaultCniResult.addressType = cns.Default } return ipamResults, nil From 9062e836e38c5d93e5f4d01fe1fd1506ba4b15e8 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 18 Sep 2023 20:48:01 +0000 Subject: [PATCH 07/64] address comments --- cni/network/invoker_azure.go | 7 ++-- cni/network/invoker_cns.go | 78 ++++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 37 deletions(-) diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index f8995b53a3..d6635bf816 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -69,7 +69,6 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er err = invoker.plugin.Errorf("Failed to allocate pool: %v", err) return addResult, err } - addResult.defaultCniResult = CNIResult{ipResult: ipv4Result, addressType: cns.Default, isDefaultInterface: true} if len(ipv4Result.IPs) > 0 { addResult.hostSubnetPrefix = ipv4Result.IPs[0].Address } @@ -101,12 +100,14 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er if err != nil { err = invoker.plugin.Errorf("Failed to allocate v6 pool: %v", err) } else { - addResult.defaultCniResult.ipResult.IPs = append(addResult.defaultCniResult.ipResult.IPs, ipv6Result.IPs...) - addResult.defaultCniResult.ipResult.Routes = append(addResult.defaultCniResult.ipResult.Routes, ipv6Result.Routes...) + ipv4Result.IPs = append(ipv4Result.IPs, ipv6Result.IPs...) + ipv4Result.Routes = append(ipv4Result.Routes, ipv6Result.Routes...) addResult.ipv6Enabled = true } } + addResult.defaultCniResult = CNIResult{ipResult: ipv4Result, addressType: cns.Default, isDefaultInterface: true} + return addResult, err } diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index b7ac8e52ff..9a34856215 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -122,7 +122,9 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } addResult := IPAMAddResult{} + // Default address type will be the default interface unless isDefaultInterface is true for a secondary address var isDefaultInterfaceSet bool + defaultRoutes := make([]*cniTypes.Route, 0) for i := 0; i < len(response.PodIPInfo); i++ { info := IPResultInfo{ @@ -177,23 +179,12 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro isDefaultInterface: info.isDefaultInterface, } - for _, route := range info.routes { - _, dst, err := net.ParseCIDR(route.IPAddress) - if err != nil { - return IPAMAddResult{}, fmt.Errorf("unable to parse destination %s: %w", route.IPAddress, err) - } - gw := net.ParseIP(route.GatewayIPAddress) - if gw == nil { - return IPAMAddResult{}, fmt.Errorf("unable to parse gateway %s: %w", route.GatewayIPAddress, err) - } - - result.ipResult.Routes = append(result.ipResult.Routes, - &cniTypes.Route{ - Dst: *dst, - GW: gw, - }) + routes, err := getRoutes(info.routes) + if err != nil { + return IPAMAddResult{}, err } + result.ipResult.Routes = append(result.ipResult.Routes, routes...) addResult.cniResults = append(addResult.cniResults, result) default: // set the NC Primary IP in options @@ -233,8 +224,9 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } if ip := net.ParseIP(info.podIPAddress); ip != nil { - if addResult.defaultCniResult.ipResult == nil { - addResult.defaultCniResult.ipResult = &cniTypesCurr.Result{} + defaultCniResult := addResult.defaultCniResult.ipResult + if defaultCniResult == nil { + defaultCniResult = &cniTypesCurr.Result{} } defaultRouteDstPrefix := network.Ipv4DefaultRouteDstPrefix @@ -243,33 +235,24 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro addResult.ipv6Enabled = true } - addResult.defaultCniResult.ipResult.IPs = append(addResult.defaultCniResult.ipResult.IPs, + defaultCniResult.IPs = append(defaultCniResult.IPs, &cniTypesCurr.IPConfig{ Address: resultIPnet, Gateway: ncgw, }) - addResult.defaultCniResult.ipResult.Routes = append(addResult.defaultCniResult.ipResult.Routes, + defaultRoutes = append(defaultRoutes, &cniTypes.Route{ Dst: defaultRouteDstPrefix, GW: ncgw, }) - for _, route := range info.routes { - _, dst, routeErr := net.ParseCIDR(route.IPAddress) - if routeErr != nil { - return IPAMAddResult{}, fmt.Errorf("unable to parse destination %s: %w", route.IPAddress, routeErr) - } - gw := net.ParseIP(route.GatewayIPAddress) - if gw == nil { - return IPAMAddResult{}, fmt.Errorf("unable to parse gateway %s: %w", route.GatewayIPAddress, routeErr) - } - - addResult.defaultCniResult.ipResult.Routes = append(addResult.defaultCniResult.ipResult.Routes, - &cniTypes.Route{ - Dst: *dst, - GW: gw, - }) + routes, err := getRoutes(info.routes) + if err != nil { + return IPAMAddResult{}, err } + + defaultCniResult.Routes = append(defaultCniResult.Routes, routes...) + addResult.defaultCniResult.ipResult = defaultCniResult } // get the name of the primary IP address @@ -292,6 +275,10 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } addResult.defaultCniResult.isDefaultInterface = !isDefaultInterfaceSet + // add default routes if none exists + if len(addResult.defaultCniResult.ipResult.Routes) == 0 { + addResult.defaultCniResult.ipResult.Routes = defaultRoutes + } return addResult, nil } @@ -426,3 +413,26 @@ func (invoker *CNSIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConf return nil } + +func getRoutes(cnsRoutes []cns.Route) ([]*cniTypes.Route, error) { + routes := make([]*cniTypes.Route, 0) + for _, route := range cnsRoutes { + _, dst, routeErr := net.ParseCIDR(route.IPAddress) + if routeErr != nil { + return nil, fmt.Errorf("unable to parse destination %s: %w", route.IPAddress, routeErr) + } + + gw := net.ParseIP(route.GatewayIPAddress) + if gw == nil { + return nil, fmt.Errorf("unable to parse gateway %s: %w", route.GatewayIPAddress, routeErr) + } + + routes = append(routes, + &cniTypes.Route{ + Dst: *dst, + GW: gw, + }) + } + + return routes, nil +} From f407551bc066c82c011ac226fc10f0e39185b69b Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 22 Sep 2023 20:01:29 +0000 Subject: [PATCH 08/64] address comments --- cni/network/invoker_azure.go | 2 +- cni/network/invoker_cns.go | 6 +++--- cni/network/invoker_cns_test.go | 6 +++--- cni/network/invoker_mock.go | 2 +- cni/network/multitenancy.go | 2 +- cni/network/multitenancy_mock.go | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index d6635bf816..ab7497c6f9 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -106,7 +106,7 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er } } - addResult.defaultCniResult = CNIResult{ipResult: ipv4Result, addressType: cns.Default, isDefaultInterface: true} + addResult.defaultCniResult = CNIResult{ipResult: ipv4Result, nicType: cns.Default, isDefaultInterface: true} return addResult, err } diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 9a34856215..115f2969a0 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -135,7 +135,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro hostSubnet: response.PodIPInfo[i].HostPrimaryIPInfo.Subnet, hostPrimaryIP: response.PodIPInfo[i].HostPrimaryIPInfo.PrimaryIP, hostGateway: response.PodIPInfo[i].HostPrimaryIPInfo.Gateway, - addressType: response.PodIPInfo[i].AddressType, + nicType: response.PodIPInfo[i].NICType, macAddress: response.PodIPInfo[i].MacAddress, isDefaultInterface: response.PodIPInfo[i].IsDefaultInterface, routes: response.PodIPInfo[i].Routes, @@ -145,7 +145,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro zap.Any("ipinfo", info), zap.Any("podInfo", podInfo)) - switch info.addressType { + switch info.nicType { case cns.Secondary: ip, ipnet, err := response.PodIPInfo[i].PodIPConfig.GetIPNet() if ip == nil { @@ -174,7 +174,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro }, }, }, - addressType: cns.Secondary, + nicType: cns.Secondary, macAddress: macAddress, isDefaultInterface: info.isDefaultInterface, } diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 981e4f48f8..842f13002e 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -270,15 +270,15 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, - AddressType: cns.Default, + NICType: cns.Default, }, { PodIPConfig: cns.IPSubnet{ IPAddress: "20.240.1.242", PrefixLength: 24, }, - AddressType: cns.Secondary, - MacAddress: "12:34:56:78:9a:bc", + NICType: cns.Secondary, + MacAddress: "12:34:56:78:9a:bc", }, }, Response: cns.Response{ diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index 845c883217..480ebb2e95 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -58,7 +58,7 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ipResult: ¤t.Result{ IPs: []*current.IPConfig{{Address: ipnet, Gateway: gwIP}}, }, - addressType: cns.Default, + nicType: cns.Default, } invoker.ipMap[ipnet.String()] = true if invoker.v6Fail { diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 6cf3bf31a5..1f49cfc3cf 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -221,7 +221,7 @@ func (m *Multitenancy) GetAllNetworkContainers( ipamResults[i].ncResponse = &ncResponses[i] ipamResults[i].hostSubnetPrefix = hostSubnetPrefixes[i] ipamResults[i].defaultCniResult.ipResult = convertToCniResult(ipamResults[i].ncResponse, ifName) - ipamResults[i].defaultCniResult.addressType = cns.Default + ipamResults[i].defaultCniResult.nicType = cns.Default } return ipamResults, err diff --git a/cni/network/multitenancy_mock.go b/cni/network/multitenancy_mock.go index d99fd50347..dd253b0889 100644 --- a/cni/network/multitenancy_mock.go +++ b/cni/network/multitenancy_mock.go @@ -159,7 +159,7 @@ func (m *MockMultitenancy) GetAllNetworkContainers( ipamResults[i].ncResponse = &cnsResponses[i] ipamResults[i].hostSubnetPrefix = ipNets[i] ipamResults[i].defaultCniResult.ipResult = convertToCniResult(ipamResults[i].ncResponse, ifName) - ipamResults[i].defaultCniResult.addressType = cns.Default + ipamResults[i].defaultCniResult.nicType = cns.Default } return ipamResults, nil From a9d8de709d2c7c6316c939f550128e803fb4a0a4 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 22 Sep 2023 22:26:19 +0000 Subject: [PATCH 09/64] refactor cns invoker per comments --- cni/network/invoker_cns.go | 263 ++++++++++++++++++++----------------- 1 file changed, 139 insertions(+), 124 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 115f2969a0..268fab7f02 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -124,7 +124,6 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro addResult := IPAMAddResult{} // Default address type will be the default interface unless isDefaultInterface is true for a secondary address var isDefaultInterfaceSet bool - defaultRoutes := make([]*cniTypes.Route, 0) for i := 0; i < len(response.PodIPInfo); i++ { info := IPResultInfo{ @@ -147,138 +146,19 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro switch info.nicType { case cns.Secondary: - ip, ipnet, err := response.PodIPInfo[i].PodIPConfig.GetIPNet() - if ip == nil { - return IPAMAddResult{}, errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") - } - - macAddress, err := net.ParseMAC(info.macAddress) - if err != nil { - return IPAMAddResult{}, errors.Wrap(err, "Invalid mac address") - } - isDefaultInterfaceSet = isDefaultInterfaceSet || info.isDefaultInterface - result := CNIResult{ - ipResult: &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { - Address: net.IPNet{ - IP: ip, - Mask: ipnet.Mask, - }, - }, - }, - Interfaces: []*cniTypesCurr.Interface{ - { - Mac: info.macAddress, - }, - }, - }, - nicType: cns.Secondary, - macAddress: macAddress, - isDefaultInterface: info.isDefaultInterface, - } - - routes, err := getRoutes(info.routes) - if err != nil { + if err := configureSecondaryAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig); err != nil { return IPAMAddResult{}, err } - - result.ipResult.Routes = append(result.ipResult.Routes, routes...) - addResult.cniResults = append(addResult.cniResults, result) default: - // set the NC Primary IP in options - // SNATIPKey is not set for ipv6 - if net.ParseIP(info.ncPrimaryIP).To4() != nil { - addConfig.options[network.SNATIPKey] = info.ncPrimaryIP - } - - ip, ncIPNet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix)) - if ip == nil { - return IPAMAddResult{}, errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") - } - - ncgw := net.ParseIP(info.ncGatewayIPAddress) - if ncgw == nil { - // TODO: Remove v4overlay and dualstackoverlay options, after 'overlay' rolls out in AKS-RP - if (invoker.ipamMode != util.V4Overlay) && (invoker.ipamMode != util.DualStackOverlay) && (invoker.ipamMode != util.Overlay) { - return IPAMAddResult{}, errors.Wrap(errInvalidArgs, "%w: Gateway address "+info.ncGatewayIPAddress+" from response is invalid") - } - - if net.ParseIP(info.podIPAddress).To4() != nil { //nolint:gocritic - ncgw, err = getOverlayGateway(ncIPNet) - if err != nil { - return IPAMAddResult{}, err - } - } else if net.ParseIP(info.podIPAddress).To16() != nil { - ncgw = net.ParseIP(overlayGatewayV6IP) - } else { - return IPAMAddResult{}, errors.Wrap(err, "No podIPAddress is found: %w") - } - } - - // construct ipnet for result - resultIPnet := net.IPNet{ - IP: ip, - Mask: ncIPNet.Mask, - } - - if ip := net.ParseIP(info.podIPAddress); ip != nil { - defaultCniResult := addResult.defaultCniResult.ipResult - if defaultCniResult == nil { - defaultCniResult = &cniTypesCurr.Result{} - } - - defaultRouteDstPrefix := network.Ipv4DefaultRouteDstPrefix - if ip.To4() == nil { - defaultRouteDstPrefix = network.Ipv6DefaultRouteDstPrefix - addResult.ipv6Enabled = true - } - - defaultCniResult.IPs = append(defaultCniResult.IPs, - &cniTypesCurr.IPConfig{ - Address: resultIPnet, - Gateway: ncgw, - }) - defaultRoutes = append(defaultRoutes, - &cniTypes.Route{ - Dst: defaultRouteDstPrefix, - GW: ncgw, - }) - - routes, err := getRoutes(info.routes) - if err != nil { - return IPAMAddResult{}, err - } - - defaultCniResult.Routes = append(defaultCniResult.Routes, routes...) - addResult.defaultCniResult.ipResult = defaultCniResult - } - - // get the name of the primary IP address - _, hostIPNet, err := net.ParseCIDR(info.hostSubnet) - if err != nil { - return IPAMAddResult{}, fmt.Errorf("unable to parse hostSubnet: %w", err) - } - - addResult.hostSubnetPrefix = *hostIPNet - - // set subnet prefix for host vm - // setHostOptions will execute if IPAM mode is not v4 overlay and not dualStackOverlay mode - // TODO: Remove v4overlay and dualstackoverlay options, after 'overlay' rolls out in AKS-RP - if (invoker.ipamMode != util.V4Overlay) && (invoker.ipamMode != util.DualStackOverlay) && (invoker.ipamMode != util.Overlay) { - if err := setHostOptions(ncIPNet, addConfig.options, &info); err != nil { - return IPAMAddResult{}, err - } + overlayMode := (invoker.ipamMode == util.V4Overlay) || (invoker.ipamMode == util.DualStackOverlay) || (invoker.ipamMode == util.Overlay) + if err := configureDefaultAddResult(&info, &addConfig, &addResult, overlayMode); err != nil { + return IPAMAddResult{}, err } } } addResult.defaultCniResult.isDefaultInterface = !isDefaultInterfaceSet - // add default routes if none exists - if len(addResult.defaultCniResult.ipResult.Routes) == 0 { - addResult.defaultCniResult.ipResult.Routes = defaultRoutes - } return addResult, nil } @@ -436,3 +316,138 @@ func getRoutes(cnsRoutes []cns.Route) ([]*cniTypes.Route, error) { return routes, nil } + +func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, addResult *IPAMAddResult, overlayMode bool) error { + // set the NC Primary IP in options + // SNATIPKey is not set for ipv6 + if net.ParseIP(info.ncPrimaryIP).To4() != nil { + addConfig.options[network.SNATIPKey] = info.ncPrimaryIP + } + + ip, ncIPNet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix)) + if ip == nil { + return errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") + } + + ncgw := net.ParseIP(info.ncGatewayIPAddress) + if ncgw == nil { + // TODO: Remove v4overlay and dualstackoverlay options, after 'overlay' rolls out in AKS-RP + if !overlayMode { + return errors.Wrap(errInvalidArgs, "%w: Gateway address "+info.ncGatewayIPAddress+" from response is invalid") + } + + if net.ParseIP(info.podIPAddress).To4() != nil { //nolint:gocritic + ncgw, err = getOverlayGateway(ncIPNet) + if err != nil { + return err + } + } else if net.ParseIP(info.podIPAddress).To16() != nil { + ncgw = net.ParseIP(overlayGatewayV6IP) + } else { + return errors.Wrap(err, "No podIPAddress is found: %w") + } + } + + // construct ipnet for result + resultIPnet := net.IPNet{ + IP: ip, + Mask: ncIPNet.Mask, + } + + if ip := net.ParseIP(info.podIPAddress); ip != nil { + defaultCniResult := addResult.defaultCniResult.ipResult + if defaultCniResult == nil { + defaultCniResult = &cniTypesCurr.Result{} + } + + defaultRouteDstPrefix := network.Ipv4DefaultRouteDstPrefix + if ip.To4() == nil { + defaultRouteDstPrefix = network.Ipv6DefaultRouteDstPrefix + addResult.ipv6Enabled = true + } + + defaultCniResult.IPs = append(defaultCniResult.IPs, + &cniTypesCurr.IPConfig{ + Address: resultIPnet, + Gateway: ncgw, + }) + + routes, err := getRoutes(info.routes) + if err != nil { + return err + } + + if len(routes) > 0 { + defaultCniResult.Routes = append(defaultCniResult.Routes, routes...) + } else { // add default routes if none are provided + defaultCniResult.Routes = append(defaultCniResult.Routes, &cniTypes.Route{ + Dst: defaultRouteDstPrefix, + GW: ncgw, + }) + } + + addResult.defaultCniResult.ipResult = defaultCniResult + } + + // get the name of the primary IP address + _, hostIPNet, err := net.ParseCIDR(info.hostSubnet) + if err != nil { + return fmt.Errorf("unable to parse hostSubnet: %w", err) + } + + addResult.hostSubnetPrefix = *hostIPNet + + // set subnet prefix for host vm + // setHostOptions will execute if IPAM mode is not v4 overlay and not dualStackOverlay mode + // TODO: Remove v4overlay and dualstackoverlay options, after 'overlay' rolls out in AKS-RP + if !overlayMode { + if err := setHostOptions(ncIPNet, addConfig.options, info); err != nil { + return err + } + } + + return nil +} + +func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, podIPConfig *cns.IPSubnet) error { + ip, ipnet, err := podIPConfig.GetIPNet() + if ip == nil { + return errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") + } + + macAddress, err := net.ParseMAC(info.macAddress) + if err != nil { + return errors.Wrap(err, "Invalid mac address") + } + + result := CNIResult{ + ipResult: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: net.IPNet{ + IP: ip, + Mask: ipnet.Mask, + }, + }, + }, + Interfaces: []*cniTypesCurr.Interface{ + { + Mac: info.macAddress, + }, + }, + }, + nicType: cns.Secondary, + macAddress: macAddress, + isDefaultInterface: info.isDefaultInterface, + } + + routes, err := getRoutes(info.routes) + if err != nil { + return err + } + + result.ipResult.Routes = append(result.ipResult.Routes, routes...) + addResult.cniResults = append(addResult.cniResults, result) + + return nil +} From f20a188498b444e35b4470e4a2ed83a8db62d132 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 25 Sep 2023 20:25:35 +0000 Subject: [PATCH 10/64] update invokers based on contract change --- cni/network/invoker_azure.go | 6 ++--- cni/network/invoker_azure_test.go | 4 ++-- cni/network/invoker_cns.go | 40 ++++++++++++++++++------------- cni/network/invoker_cns_test.go | 16 ++++++------- cni/network/invoker_mock.go | 6 ++--- cni/network/multitenancy.go | 4 ++-- cni/network/multitenancy_mock.go | 4 ++-- 7 files changed, 43 insertions(+), 37 deletions(-) diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index ab7497c6f9..c003eb374f 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -75,8 +75,8 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er defer func() { if err != nil { - if len(addResult.defaultCniResult.ipResult.IPs) > 0 { - if er := invoker.Delete(&addResult.defaultCniResult.ipResult.IPs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { + if len(addResult.defaultInterfaceInfo.ipResult.IPs) > 0 { + if er := invoker.Delete(&addResult.defaultInterfaceInfo.ipResult.IPs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { err = invoker.plugin.Errorf("Failed to clean up IP's during Delete with error %v, after Add failed with error %w", er, err) } } else { @@ -106,7 +106,7 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er } } - addResult.defaultCniResult = CNIResult{ipResult: ipv4Result, nicType: cns.Default, isDefaultInterface: true} + addResult.defaultInterfaceInfo = InterfaceInfo{ipResult: ipv4Result, nicType: cns.Infra} return addResult, err } diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index 51ce194c8b..457eb1be89 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -205,8 +205,8 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { require.Nil(err) } - fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ipamAddResult.defaultCniResult.ipResult) - require.Exactly(tt.want, ipamAddResult.defaultCniResult.ipResult) + fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ipamAddResult.defaultInterfaceInfo.ipResult) + require.Exactly(tt.want, ipamAddResult.defaultInterfaceInfo.ipResult) }) } } diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 268fab7f02..3642b94cf9 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -122,8 +122,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } addResult := IPAMAddResult{} - // Default address type will be the default interface unless isDefaultInterface is true for a secondary address - var isDefaultInterfaceSet bool + numInterfacesWithDefaultRoutes := 0 for i := 0; i < len(response.PodIPInfo); i++ { info := IPResultInfo{ @@ -136,7 +135,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro hostGateway: response.PodIPInfo[i].HostPrimaryIPInfo.Gateway, nicType: response.PodIPInfo[i].NICType, macAddress: response.PodIPInfo[i].MacAddress, - isDefaultInterface: response.PodIPInfo[i].IsDefaultInterface, + skipDefaultRoutes: response.PodIPInfo[i].SkipDefaultRoutes, routes: response.PodIPInfo[i].Routes, } @@ -144,9 +143,12 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro zap.Any("ipinfo", info), zap.Any("podInfo", podInfo)) + if !info.skipDefaultRoutes { + numInterfacesWithDefaultRoutes += 1 + } + switch info.nicType { case cns.Secondary: - isDefaultInterfaceSet = isDefaultInterfaceSet || info.isDefaultInterface if err := configureSecondaryAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig); err != nil { return IPAMAddResult{}, err } @@ -158,7 +160,10 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } } - addResult.defaultCniResult.isDefaultInterface = !isDefaultInterfaceSet + // Make sure default routes exist for 1 interface + if numInterfacesWithDefaultRoutes != 1 { + return IPAMAddResult{}, errors.New("Add result requires an interface with default routes") + } return addResult, nil } @@ -355,9 +360,9 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } if ip := net.ParseIP(info.podIPAddress); ip != nil { - defaultCniResult := addResult.defaultCniResult.ipResult - if defaultCniResult == nil { - defaultCniResult = &cniTypesCurr.Result{} + defaultInterfaceInfo := addResult.defaultInterfaceInfo.ipResult + if defaultInterfaceInfo == nil { + defaultInterfaceInfo = &cniTypesCurr.Result{} } defaultRouteDstPrefix := network.Ipv4DefaultRouteDstPrefix @@ -366,7 +371,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add addResult.ipv6Enabled = true } - defaultCniResult.IPs = append(defaultCniResult.IPs, + defaultInterfaceInfo.IPs = append(defaultInterfaceInfo.IPs, &cniTypesCurr.IPConfig{ Address: resultIPnet, Gateway: ncgw, @@ -378,15 +383,16 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } if len(routes) > 0 { - defaultCniResult.Routes = append(defaultCniResult.Routes, routes...) + defaultInterfaceInfo.Routes = append(defaultInterfaceInfo.Routes, routes...) } else { // add default routes if none are provided - defaultCniResult.Routes = append(defaultCniResult.Routes, &cniTypes.Route{ + defaultInterfaceInfo.Routes = append(defaultInterfaceInfo.Routes, &cniTypes.Route{ Dst: defaultRouteDstPrefix, GW: ncgw, }) } - addResult.defaultCniResult.ipResult = defaultCniResult + addResult.defaultInterfaceInfo.ipResult = defaultInterfaceInfo + addResult.defaultInterfaceInfo.skipDefaultRoutes = info.skipDefaultRoutes } // get the name of the primary IP address @@ -420,7 +426,7 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p return errors.Wrap(err, "Invalid mac address") } - result := CNIResult{ + result := InterfaceInfo{ ipResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { @@ -436,9 +442,9 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p }, }, }, - nicType: cns.Secondary, - macAddress: macAddress, - isDefaultInterface: info.isDefaultInterface, + nicType: cns.Secondary, + macAddress: macAddress, + skipDefaultRoutes: info.skipDefaultRoutes, } routes, err := getRoutes(info.routes) @@ -447,7 +453,7 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p } result.ipResult.Routes = append(result.ipResult.Routes, routes...) - addResult.cniResults = append(addResult.cniResults, result) + addResult.secondaryInterfaceInfo = append(addResult.secondaryInterfaceInfo, result) return nil } diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 842f13002e..0b4b0b2fc9 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -270,7 +270,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, - NICType: cns.Default, + NICType: cns.Infra, }, { PodIPConfig: cns.IPSubnet{ @@ -347,10 +347,10 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { require.NoError(err) } - fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.cniResults) - require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultCniResult.ipResult, "incorrect default response") + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.secondaryInterfaceInfo) + require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultInterfaceInfo.ipResult, "incorrect default response") if tt.wantMultitenantResult != nil { - require.Equalf(tt.wantMultitenantResult, ipamAddResult.cniResults[0].ipResult, "incorrect multitenant response") + require.Equalf(tt.wantMultitenantResult, ipamAddResult.secondaryInterfaceInfo[0].ipResult, "incorrect multitenant response") } }) } @@ -573,10 +573,10 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { require.NoError(err) } - fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.cniResults) - require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultCniResult.ipResult, "incorrect default response") + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.secondaryInterfaceInfo) + require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultInterfaceInfo.ipResult, "incorrect default response") if tt.wantMultitenantResult != nil { - require.Equalf(tt.wantMultitenantResult, ipamAddResult.cniResults[0].ipResult, "incorrect multitenant response") + require.Equalf(tt.wantMultitenantResult, ipamAddResult.secondaryInterfaceInfo[0].ipResult, "incorrect multitenant response") } }) } @@ -692,7 +692,7 @@ func TestCNSIPAMInvoker_Add_UnsupportedAPI(t *testing.T) { t.Fatalf("expected an error %+v but none received", err) } require.NoError(err) - require.Equalf(tt.want, ipamAddResult.defaultCniResult.ipResult, "incorrect ipv4 response") + require.Equalf(tt.want, ipamAddResult.defaultInterfaceInfo.ipResult, "incorrect ipv4 response") }) } } diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index 480ebb2e95..2820b78da7 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -54,11 +54,11 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ip := net.ParseIP(ipv4Str) ipnet := net.IPNet{IP: ip, Mask: net.CIDRMask(subnetBits, ipv4Bits)} gwIP := net.ParseIP("10.240.0.1") - ipamAddResult.defaultCniResult = CNIResult{ + ipamAddResult.defaultInterfaceInfo = InterfaceInfo{ ipResult: ¤t.Result{ IPs: []*current.IPConfig{{Address: ipnet, Gateway: gwIP}}, }, - nicType: cns.Default, + nicType: cns.Infra, } invoker.ipMap[ipnet.String()] = true if invoker.v6Fail { @@ -74,7 +74,7 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ip := net.ParseIP(ipv6Str) ipnet := net.IPNet{IP: ip, Mask: net.CIDRMask(subnetv6Bits, ipv6Bits)} gwIP := net.ParseIP("fc00::1") - ipamAddResult.defaultCniResult.ipResult.IPs = append(ipamAddResult.defaultCniResult.ipResult.IPs, ¤t.IPConfig{Address: ipnet, Gateway: gwIP}) + ipamAddResult.defaultInterfaceInfo.ipResult.IPs = append(ipamAddResult.defaultInterfaceInfo.ipResult.IPs, ¤t.IPConfig{Address: ipnet, Gateway: gwIP}) invoker.ipMap[ipnet.String()] = true } diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 1f49cfc3cf..9e9a9d11dd 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -220,8 +220,8 @@ func (m *Multitenancy) GetAllNetworkContainers( for i := 0; i < len(ncResponses); i++ { ipamResults[i].ncResponse = &ncResponses[i] ipamResults[i].hostSubnetPrefix = hostSubnetPrefixes[i] - ipamResults[i].defaultCniResult.ipResult = convertToCniResult(ipamResults[i].ncResponse, ifName) - ipamResults[i].defaultCniResult.nicType = cns.Default + ipamResults[i].defaultInterfaceInfo.ipResult = convertToCniResult(ipamResults[i].ncResponse, ifName) + ipamResults[i].defaultInterfaceInfo.nicType = cns.Infra } return ipamResults, err diff --git a/cni/network/multitenancy_mock.go b/cni/network/multitenancy_mock.go index dd253b0889..007865fdd8 100644 --- a/cni/network/multitenancy_mock.go +++ b/cni/network/multitenancy_mock.go @@ -158,8 +158,8 @@ func (m *MockMultitenancy) GetAllNetworkContainers( for i := 0; i < len(cnsResponses); i++ { ipamResults[i].ncResponse = &cnsResponses[i] ipamResults[i].hostSubnetPrefix = ipNets[i] - ipamResults[i].defaultCniResult.ipResult = convertToCniResult(ipamResults[i].ncResponse, ifName) - ipamResults[i].defaultCniResult.nicType = cns.Default + ipamResults[i].defaultInterfaceInfo.ipResult = convertToCniResult(ipamResults[i].ncResponse, ifName) + ipamResults[i].defaultInterfaceInfo.nicType = cns.Infra } return ipamResults, nil From 6c356e53b06d317e9979463c1b85f67f6762f6d8 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 25 Sep 2023 22:08:45 +0000 Subject: [PATCH 11/64] update test --- cni/network/invoker_cns.go | 15 ++++++++++----- cni/network/invoker_cns_test.go | 3 ++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 3642b94cf9..638e763df9 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -143,16 +143,21 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro zap.Any("ipinfo", info), zap.Any("podInfo", podInfo)) - if !info.skipDefaultRoutes { - numInterfacesWithDefaultRoutes += 1 - } - switch info.nicType { case cns.Secondary: + if !info.skipDefaultRoutes { + numInterfacesWithDefaultRoutes += 1 + } + if err := configureSecondaryAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig); err != nil { return IPAMAddResult{}, err } default: + // only count dualstack interface once + if addResult.defaultInterfaceInfo.ipResult == nil && !info.skipDefaultRoutes { + numInterfacesWithDefaultRoutes += 1 + } + overlayMode := (invoker.ipamMode == util.V4Overlay) || (invoker.ipamMode == util.DualStackOverlay) || (invoker.ipamMode == util.Overlay) if err := configureDefaultAddResult(&info, &addConfig, &addResult, overlayMode); err != nil { return IPAMAddResult{}, err @@ -162,7 +167,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro // Make sure default routes exist for 1 interface if numInterfacesWithDefaultRoutes != 1 { - return IPAMAddResult{}, errors.New("Add result requires an interface with default routes") + return IPAMAddResult{}, errors.New("Add result requires exactly one interface with default routes") } return addResult, nil diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 0b4b0b2fc9..293da96dc9 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -270,7 +270,8 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, - NICType: cns.Infra, + NICType: cns.Infra, + SkipDefaultRoutes: true, }, { PodIPConfig: cns.IPSubnet{ From 426e723602bbb2ca884a6ab2ca840bd10dde1310 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 27 Sep 2023 19:07:43 +0000 Subject: [PATCH 12/64] update with contract changes --- cni/network/invoker.go | 2 +- cni/network/invoker_azure.go | 2 +- cni/network/invoker_cns.go | 6 +++--- cni/network/invoker_cns_test.go | 4 ++-- cni/network/invoker_mock.go | 2 +- cni/network/multitenancy.go | 2 +- cni/network/multitenancy_mock.go | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index 78f87f14a6..f54f58fe2e 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -38,7 +38,7 @@ type IPAMAddResult struct { type InterfaceInfo struct { ipResult *cniTypesCurr.Result - nicType string + nicType cns.NICType macAddress net.HardwareAddr skipDefaultRoutes bool } diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index c003eb374f..5e3962bb47 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -106,7 +106,7 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er } } - addResult.defaultInterfaceInfo = InterfaceInfo{ipResult: ipv4Result, nicType: cns.Infra} + addResult.defaultInterfaceInfo = InterfaceInfo{ipResult: ipv4Result, nicType: cns.InfraNIC} return addResult, err } diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 638e763df9..a67422d59a 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -44,7 +44,7 @@ type IPResultInfo struct { hostSubnet string hostPrimaryIP string hostGateway string - nicType string + nicType cns.NICType macAddress string skipDefaultRoutes bool routes []cns.Route @@ -144,7 +144,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro zap.Any("podInfo", podInfo)) switch info.nicType { - case cns.Secondary: + case cns.DelegatedVMNIC: if !info.skipDefaultRoutes { numInterfacesWithDefaultRoutes += 1 } @@ -447,7 +447,7 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p }, }, }, - nicType: cns.Secondary, + nicType: cns.DelegatedVMNIC, macAddress: macAddress, skipDefaultRoutes: info.skipDefaultRoutes, } diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 293da96dc9..055c492a79 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -270,7 +270,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, - NICType: cns.Infra, + NICType: cns.InfraNIC, SkipDefaultRoutes: true, }, { @@ -278,7 +278,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { IPAddress: "20.240.1.242", PrefixLength: 24, }, - NICType: cns.Secondary, + NICType: cns.DelegatedVMNIC, MacAddress: "12:34:56:78:9a:bc", }, }, diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index 2820b78da7..5b92ed7950 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -58,7 +58,7 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ipResult: ¤t.Result{ IPs: []*current.IPConfig{{Address: ipnet, Gateway: gwIP}}, }, - nicType: cns.Infra, + nicType: cns.InfraNIC, } invoker.ipMap[ipnet.String()] = true if invoker.v6Fail { diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 9e9a9d11dd..4dd09f81c1 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -221,7 +221,7 @@ func (m *Multitenancy) GetAllNetworkContainers( ipamResults[i].ncResponse = &ncResponses[i] ipamResults[i].hostSubnetPrefix = hostSubnetPrefixes[i] ipamResults[i].defaultInterfaceInfo.ipResult = convertToCniResult(ipamResults[i].ncResponse, ifName) - ipamResults[i].defaultInterfaceInfo.nicType = cns.Infra + ipamResults[i].defaultInterfaceInfo.nicType = cns.InfraNIC } return ipamResults, err diff --git a/cni/network/multitenancy_mock.go b/cni/network/multitenancy_mock.go index 007865fdd8..c979bd597d 100644 --- a/cni/network/multitenancy_mock.go +++ b/cni/network/multitenancy_mock.go @@ -159,7 +159,7 @@ func (m *MockMultitenancy) GetAllNetworkContainers( ipamResults[i].ncResponse = &cnsResponses[i] ipamResults[i].hostSubnetPrefix = ipNets[i] ipamResults[i].defaultInterfaceInfo.ipResult = convertToCniResult(ipamResults[i].ncResponse, ifName) - ipamResults[i].defaultInterfaceInfo.nicType = cns.Infra + ipamResults[i].defaultInterfaceInfo.nicType = cns.InfraNIC } return ipamResults, nil From 28875e82741f0c82472f6756bb9dcc8e6463fe6f Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 27 Sep 2023 20:23:11 +0000 Subject: [PATCH 13/64] fix linter errs --- cni/network/invoker_cns.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index a67422d59a..2bb7a2de99 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -146,16 +146,18 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro switch info.nicType { case cns.DelegatedVMNIC: if !info.skipDefaultRoutes { - numInterfacesWithDefaultRoutes += 1 + numInterfacesWithDefaultRoutes++ } if err := configureSecondaryAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig); err != nil { return IPAMAddResult{}, err } + case cns.InfraNIC: + fallthrough default: // only count dualstack interface once if addResult.defaultInterfaceInfo.ipResult == nil && !info.skipDefaultRoutes { - numInterfacesWithDefaultRoutes += 1 + numInterfacesWithDefaultRoutes++ } overlayMode := (invoker.ipamMode == util.V4Overlay) || (invoker.ipamMode == util.DualStackOverlay) || (invoker.ipamMode == util.Overlay) @@ -407,6 +409,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } addResult.hostSubnetPrefix = *hostIPNet + addResult.defaultInterfaceInfo.nicType = cns.InfraNIC // set subnet prefix for host vm // setHostOptions will execute if IPAM mode is not v4 overlay and not dualStackOverlay mode From e37ae64decceaf9f71be2c0db14be88a65a4297a Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 27 Sep 2023 20:37:29 +0000 Subject: [PATCH 14/64] fix naming --- cni/network/invoker.go | 6 +++--- cni/network/invoker_cns.go | 2 +- cni/network/invoker_cns_test.go | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cni/network/invoker.go b/cni/network/invoker.go index f54f58fe2e..453f0a1ca0 100644 --- a/cni/network/invoker.go +++ b/cni/network/invoker.go @@ -27,9 +27,9 @@ type IPAMAddConfig struct { } type IPAMAddResult struct { - // Splitting defaultInterfaceInfo from secondaryInterfaceInfo so we don't need to loop for default CNI result every time - defaultInterfaceInfo InterfaceInfo - secondaryInterfaceInfo []InterfaceInfo + // Splitting defaultInterfaceInfo from secondaryInterfacesInfo so we don't need to loop for default CNI result every time + defaultInterfaceInfo InterfaceInfo + secondaryInterfacesInfo []InterfaceInfo // ncResponse is used for Swift 1.0 multitenancy ncResponse *cns.GetNetworkContainerResponse hostSubnetPrefix net.IPNet diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 2bb7a2de99..c07da9ed59 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -461,7 +461,7 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p } result.ipResult.Routes = append(result.ipResult.Routes, routes...) - addResult.secondaryInterfaceInfo = append(addResult.secondaryInterfaceInfo, result) + addResult.secondaryInterfacesInfo = append(addResult.secondaryInterfacesInfo, result) return nil } diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 055c492a79..7fda0bcb08 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -348,10 +348,10 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { require.NoError(err) } - fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.secondaryInterfaceInfo) + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.secondaryInterfacesInfo) require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultInterfaceInfo.ipResult, "incorrect default response") if tt.wantMultitenantResult != nil { - require.Equalf(tt.wantMultitenantResult, ipamAddResult.secondaryInterfaceInfo[0].ipResult, "incorrect multitenant response") + require.Equalf(tt.wantMultitenantResult, ipamAddResult.secondaryInterfacesInfo[0].ipResult, "incorrect multitenant response") } }) } @@ -574,10 +574,10 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { require.NoError(err) } - fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.secondaryInterfaceInfo) + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.secondaryInterfacesInfo) require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultInterfaceInfo.ipResult, "incorrect default response") if tt.wantMultitenantResult != nil { - require.Equalf(tt.wantMultitenantResult, ipamAddResult.secondaryInterfaceInfo[0].ipResult, "incorrect multitenant response") + require.Equalf(tt.wantMultitenantResult, ipamAddResult.secondaryInterfacesInfo[0].ipResult, "incorrect multitenant response") } }) } From d8fd006527a871574be08f6f4564a60a8a108d93 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 28 Sep 2023 15:28:27 +0000 Subject: [PATCH 15/64] fix linter --- cni/network/invoker_cns.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index c07da9ed59..0887aaedab 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -152,8 +152,6 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro if err := configureSecondaryAddResult(&info, &addResult, &response.PodIPInfo[i].PodIPConfig); err != nil { return IPAMAddResult{}, err } - case cns.InfraNIC: - fallthrough default: // only count dualstack interface once if addResult.defaultInterfaceInfo.ipResult == nil && !info.skipDefaultRoutes { @@ -314,15 +312,10 @@ func getRoutes(cnsRoutes []cns.Route) ([]*cniTypes.Route, error) { return nil, fmt.Errorf("unable to parse destination %s: %w", route.IPAddress, routeErr) } - gw := net.ParseIP(route.GatewayIPAddress) - if gw == nil { - return nil, fmt.Errorf("unable to parse gateway %s: %w", route.GatewayIPAddress, routeErr) - } - routes = append(routes, &cniTypes.Route{ Dst: *dst, - GW: gw, + GW: net.ParseIP(route.GatewayIPAddress), }) } From 6c0987bdfb14adc95db3ae78548120fec05f7a7f Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 28 Sep 2023 16:19:14 +0000 Subject: [PATCH 16/64] fix linter --- cni/network/invoker_cns.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 0887aaedab..9484b81665 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -143,6 +143,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro zap.Any("ipinfo", info), zap.Any("podInfo", podInfo)) + //nolint:exhaustive // ignore exhaustive types check switch info.nicType { case cns.DelegatedVMNIC: if !info.skipDefaultRoutes { From 50f82f107c9be49ebb12a005f372b2eecce4bf99 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Tue, 3 Oct 2023 17:02:31 +0000 Subject: [PATCH 17/64] address comments --- cni/network/invoker_azure.go | 12 ++++---- cni/network/invoker_azure_test.go | 46 +++++++++++++++++++++++-------- cni/network/invoker_cns.go | 27 +++++++----------- 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index 5e3962bb47..f26d798f40 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -58,7 +58,7 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er } // Call into IPAM plugin to allocate an address pool for the network. - ipv4Result, err := invoker.plugin.DelegateAdd(addConfig.nwCfg.IPAM.Type, addConfig.nwCfg) + result, err := invoker.plugin.DelegateAdd(addConfig.nwCfg.IPAM.Type, addConfig.nwCfg) if err != nil && strings.Contains(err.Error(), ipam.ErrNoAvailableAddressPools.Error()) { invoker.deleteIpamState() logger.Info("Retry pool allocation after deleting IPAM state") @@ -69,8 +69,8 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er err = invoker.plugin.Errorf("Failed to allocate pool: %v", err) return addResult, err } - if len(ipv4Result.IPs) > 0 { - addResult.hostSubnetPrefix = ipv4Result.IPs[0].Address + if len(result.IPs) > 0 { + addResult.hostSubnetPrefix = result.IPs[0].Address } defer func() { @@ -100,13 +100,13 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er if err != nil { err = invoker.plugin.Errorf("Failed to allocate v6 pool: %v", err) } else { - ipv4Result.IPs = append(ipv4Result.IPs, ipv6Result.IPs...) - ipv4Result.Routes = append(ipv4Result.Routes, ipv6Result.Routes...) + result.IPs = append(result.IPs, ipv6Result.IPs...) + result.Routes = append(result.Routes, ipv6Result.Routes...) addResult.ipv6Enabled = true } } - addResult.defaultInterfaceInfo = InterfaceInfo{ipResult: ipv4Result, nicType: cns.InfraNIC} + addResult.defaultInterfaceInfo = InterfaceInfo{ipResult: result, nicType: cns.InfraNIC} return addResult, err } diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index 457eb1be89..8fc27276c6 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -22,10 +22,12 @@ type mockDelegatePlugin struct { } type add struct { - resultsIPv4 *cniTypesCurr.Result - resultsIPv6 *cniTypesCurr.Result - errv4 error - errv6 error + resultsIPv4 []*cniTypesCurr.Result + resultsIPv6 []*cniTypesCurr.Result + resultsIPv4Index int + resultsIPv6Index int + errv4 error + errv6 error } func (d *add) DelegateAdd(pluginName string, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { @@ -33,13 +35,23 @@ func (d *add) DelegateAdd(pluginName string, nwCfg *cni.NetworkConfig) (*cniType if d.errv6 != nil { return nil, d.errv6 } - return d.resultsIPv6, nil + if d.resultsIPv6 == nil || d.resultsIPv6Index-1 > len(d.resultsIPv6) { + return nil, errors.New("no more ipv6 results in mock available") //nolint:goerr113 + } + res := d.resultsIPv6[d.resultsIPv6Index] + d.resultsIPv6Index++ + return res, nil } if d.errv4 != nil { return nil, d.errv4 } - return d.resultsIPv4, nil + if d.resultsIPv4 == nil || d.resultsIPv4Index-1 > len(d.resultsIPv4) { + return nil, errors.New("no more ipv4 results in mock available") //nolint:goerr113 + } + res := d.resultsIPv4[d.resultsIPv4Index] + d.resultsIPv4Index++ + return res, nil } type del struct { @@ -70,6 +82,18 @@ func getCIDRNotationForAddress(ipaddresswithcidr string) *net.IPNet { return ipnet } +func getSingleResult(ip string) []*cniTypesCurr.Result { + return []*cniTypesCurr.Result{ + { + IPs: []*cniTypesCurr.IPConfig{ + { + Address: *getCIDRNotationForAddress(ip), + }, + }, + }, + } +} + func getResult(ips ...string) *cniTypesCurr.Result { res := &cniTypesCurr.Result{} for _, ip := range ips { @@ -117,7 +141,7 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { fields: fields{ plugin: &mockDelegatePlugin{ add: add{ - resultsIPv4: getResult("10.0.0.1/24"), + resultsIPv4: getSingleResult("10.0.0.1/24"), }, del: del{}, }, @@ -135,8 +159,8 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { fields: fields{ plugin: &mockDelegatePlugin{ add: add{ - resultsIPv4: getResult("10.0.0.1/24"), - resultsIPv6: getResult("2001:0db8:abcd:0015::0/64"), + resultsIPv4: getSingleResult("10.0.0.1/24"), + resultsIPv6: getSingleResult("2001:0db8:abcd:0015::0/64"), }, }, nwInfo: getNwInfo("10.0.0.0/24", "2001:db8:abcd:0012::0/64"), @@ -171,7 +195,7 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { fields: fields{ plugin: &mockDelegatePlugin{ add: add{ - resultsIPv4: getResult("10.0.0.1/24"), + resultsIPv4: getSingleResult("10.0.0.1/24"), errv6: errors.New("test v6 error"), //nolint:goerr113 }, }, @@ -372,7 +396,7 @@ func TestRemoveIpamState_Add(t *testing.T) { fields: fields{ plugin: &mockDelegatePlugin{ add: add{ - resultsIPv4: getResult("10.0.0.1/24"), + resultsIPv4: getSingleResult("10.0.0.1/24"), errv4: ipam.ErrNoAvailableAddressPools, }, }, diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 9484b81665..acf455743c 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -22,10 +22,11 @@ import ( ) var ( - errEmptyCNIArgs = errors.New("empty CNI cmd args not allowed") - errInvalidArgs = errors.New("invalid arg(s)") - overlayGatewayV6IP = "fe80::1234:5678:9abc" - watcherPath = "/var/run/azure-vnet/deleteIDs" + errEmptyCNIArgs = errors.New("empty CNI cmd args not allowed") + errInvalidArgs = errors.New("invalid arg(s)") + errInvalidDefaultRouting = errors.New("add result requires exactly one interface with default routes") + overlayGatewayV6IP = "fe80::1234:5678:9abc" + watcherPath = "/var/run/azure-vnet/deleteIDs" ) type CNSIPAMInvoker struct { @@ -168,7 +169,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro // Make sure default routes exist for 1 interface if numInterfacesWithDefaultRoutes != 1 { - return IPAMAddResult{}, errors.New("Add result requires exactly one interface with default routes") + return IPAMAddResult{}, errInvalidDefaultRouting } return addResult, nil @@ -354,12 +355,6 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add } } - // construct ipnet for result - resultIPnet := net.IPNet{ - IP: ip, - Mask: ncIPNet.Mask, - } - if ip := net.ParseIP(info.podIPAddress); ip != nil { defaultInterfaceInfo := addResult.defaultInterfaceInfo.ipResult if defaultInterfaceInfo == nil { @@ -374,7 +369,10 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add defaultInterfaceInfo.IPs = append(defaultInterfaceInfo.IPs, &cniTypesCurr.IPConfig{ - Address: resultIPnet, + Address: net.IPNet{ + IP: ip, + Mask: ncIPNet.Mask, + }, Gateway: ncgw, }) @@ -438,11 +436,6 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p }, }, }, - Interfaces: []*cniTypesCurr.Interface{ - { - Mac: info.macAddress, - }, - }, }, nicType: cns.DelegatedVMNIC, macAddress: macAddress, From c12d9527d4e1470ac578bb78bcd45f325ccf3151 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Tue, 3 Oct 2023 17:45:44 +0000 Subject: [PATCH 18/64] update tests --- cni/network/invoker_cns_test.go | 40 +++++++++++++++++---------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 7fda0bcb08..7e4e01ff5c 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -51,6 +51,9 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { unsupportedAPIs := make(map[cnsAPIName]struct{}) unsupportedAPIs["RequestIPs"] = struct{}{} + macAddress := "12:34:56:78:9a:bc" + parsedMacAddress, _ := net.ParseMAC(macAddress) + type fields struct { podName string podNamespace string @@ -65,12 +68,12 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { } tests := []struct { - name string - fields fields - args args - wantDefaultResult *cniTypesCurr.Result - wantMultitenantResult *cniTypesCurr.Result - wantErr bool + name string + fields fields + args args + wantDefaultResult *cniTypesCurr.Result + wantSecondaryInterfacesInfo InterfaceInfo + wantErr bool }{ { name: "Test happy CNI Overlay add in v4overlay ipamMode", @@ -279,7 +282,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { PrefixLength: 24, }, NICType: cns.DelegatedVMNIC, - MacAddress: "12:34:56:78:9a:bc", + MacAddress: macAddress, }, }, Response: cns.Response{ @@ -315,17 +318,16 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { }, }, }, - wantMultitenantResult: &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { - Address: *getCIDRNotationForAddress("20.240.1.242/24"), - }, - }, - Interfaces: []*cniTypesCurr.Interface{ - { - Mac: "12:34:56:78:9a:bc", + wantSecondaryInterfacesInfo: InterfaceInfo{ + ipResult: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: *getCIDRNotationForAddress("20.240.1.242/24"), + }, }, }, + nicType: cns.DelegatedVMNIC, + macAddress: parsedMacAddress, }, wantErr: false, }, @@ -348,10 +350,10 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { require.NoError(err) } - fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.secondaryInterfacesInfo) + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantSecondaryInterfacesInfo, ipamAddResult.secondaryInterfacesInfo) require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultInterfaceInfo.ipResult, "incorrect default response") - if tt.wantMultitenantResult != nil { - require.Equalf(tt.wantMultitenantResult, ipamAddResult.secondaryInterfacesInfo[0].ipResult, "incorrect multitenant response") + if tt.wantSecondaryInterfacesInfo.ipResult != nil { + require.EqualValues(tt.wantSecondaryInterfacesInfo, ipamAddResult.secondaryInterfacesInfo[0], "incorrect multitenant response") } }) } From 27e9e7706ec38bd860e87ada2d375bcd25e1938c Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 4 Oct 2023 19:02:53 +0000 Subject: [PATCH 19/64] add tests --- cni/network/invoker_cns.go | 18 ++-- cni/network/invoker_cns_test.go | 134 ++++++++++++++++++++++++++++ cni/network/invoker_mock.go | 44 ++++++--- cni/network/multitenancy_test.go | 12 +-- cni/network/network_test.go | 90 +++++++++++++++++-- cni/network/network_windows_test.go | 14 +-- 6 files changed, 272 insertions(+), 40 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index acf455743c..909fea82b0 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -25,6 +25,7 @@ var ( errEmptyCNIArgs = errors.New("empty CNI cmd args not allowed") errInvalidArgs = errors.New("invalid arg(s)") errInvalidDefaultRouting = errors.New("add result requires exactly one interface with default routes") + errInvalidGatewayIP = errors.New("invalid gateway IP") overlayGatewayV6IP = "fe80::1234:5678:9abc" watcherPath = "/var/run/azure-vnet/deleteIDs" ) @@ -306,7 +307,7 @@ func (invoker *CNSIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConf return nil } -func getRoutes(cnsRoutes []cns.Route) ([]*cniTypes.Route, error) { +func getRoutes(cnsRoutes []cns.Route, skipDefaultRoutes bool) ([]*cniTypes.Route, error) { routes := make([]*cniTypes.Route, 0) for _, route := range cnsRoutes { _, dst, routeErr := net.ParseCIDR(route.IPAddress) @@ -314,10 +315,15 @@ func getRoutes(cnsRoutes []cns.Route) ([]*cniTypes.Route, error) { return nil, fmt.Errorf("unable to parse destination %s: %w", route.IPAddress, routeErr) } + gw := net.ParseIP(route.GatewayIPAddress) + if gw == nil && skipDefaultRoutes { + return nil, errors.Wrap(errInvalidGatewayIP, route.GatewayIPAddress) + } + routes = append(routes, &cniTypes.Route{ Dst: *dst, - GW: net.ParseIP(route.GatewayIPAddress), + GW: gw, }) } @@ -376,9 +382,9 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add Gateway: ncgw, }) - routes, err := getRoutes(info.routes) - if err != nil { - return err + routes, getRoutesErr := getRoutes(info.routes, info.skipDefaultRoutes) + if getRoutesErr != nil { + return getRoutesErr } if len(routes) > 0 { @@ -442,7 +448,7 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p skipDefaultRoutes: info.skipDefaultRoutes, } - routes, err := getRoutes(info.routes) + routes, err := getRoutes(info.routes, info.skipDefaultRoutes) if err != nil { return err } diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 7e4e01ff5c..0b569e08ac 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -331,6 +331,140 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { }, wantErr: false, }, + { + name: "Test fail CNI add with invalid mac in delegated VM nic response", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + ipamMode: util.Overlay, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: cns.IPConfigsRequest{ + PodInterfaceID: "testcont-testifname3", + InfraContainerID: "testcontainerid3", + OrchestratorContext: marshallPodInfo(testPodInfo), + }, + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + DNSServers: nil, + GatewayIPAddress: "10.0.0.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", + }, + NICType: cns.InfraNIC, + SkipDefaultRoutes: true, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "20.240.1.242", + PrefixLength: 24, + }, + NICType: cns.DelegatedVMNIC, + MacAddress: "bad mac", + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid3", + Netns: "testnetns3", + IfName: "testifname3", + }, + hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), + options: map[string]interface{}{}, + }, + wantErr: true, + }, + { + name: "Test fail CNI add with invalid ip config in delegated VM nic response", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + ipamMode: util.Overlay, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: cns.IPConfigsRequest{ + PodInterfaceID: "testcont-testifname3", + InfraContainerID: "testcontainerid3", + OrchestratorContext: marshallPodInfo(testPodInfo), + }, + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + DNSServers: nil, + GatewayIPAddress: "10.0.0.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", + }, + NICType: cns.InfraNIC, + SkipDefaultRoutes: true, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "bad ip", + PrefixLength: 24, + }, + NICType: cns.DelegatedVMNIC, + MacAddress: macAddress, + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid3", + Netns: "testnetns3", + IfName: "testifname3", + }, + hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), + options: map[string]interface{}{}, + }, + wantErr: true, + }, } for _, tt := range tests { tt := tt diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index 5b92ed7950..98f081e737 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -18,24 +18,30 @@ const ( ) var ( - errV4 = errors.New("v4 fail") - errV6 = errors.New("v6 Fail") - errDeleteIpam = errors.New("delete fail") + errV4 = errors.New("v4 fail") + errV6 = errors.New("v6 Fail") + errDelegatedVMNIC = errors.New("delegatedVMNIC fail") + errDeleteIpam = errors.New("delete fail") ) type MockIpamInvoker struct { - isIPv6 bool - v4Fail bool - v6Fail bool - ipMap map[string]bool + isIPv6 bool + v4Fail bool + v6Fail bool + delegatedVMNIC bool + delegatedVMNICFail bool + ipMap map[string]bool } -func NewMockIpamInvoker(ipv6, v4Fail, v6Fail bool) *MockIpamInvoker { +func NewMockIpamInvoker(ipv6, v4Fail, v6Fail, delegatedVMNIC, delegatedVMNICFail bool) *MockIpamInvoker { return &MockIpamInvoker{ - isIPv6: ipv6, - v4Fail: v4Fail, - v6Fail: v6Fail, - ipMap: make(map[string]bool), + isIPv6: ipv6, + v4Fail: v4Fail, + v6Fail: v6Fail, + delegatedVMNIC: delegatedVMNIC, + delegatedVMNICFail: delegatedVMNICFail, + + ipMap: make(map[string]bool), } } @@ -78,6 +84,20 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes invoker.ipMap[ipnet.String()] = true } + if invoker.delegatedVMNIC { + if invoker.delegatedVMNICFail { + return IPAMAddResult{}, errDelegatedVMNIC + } + + ipStr := "20.20.20.20/32" + _, ipnet, _ := net.ParseCIDR(ipStr) + ipamAddResult.secondaryInterfacesInfo = append(ipamAddResult.secondaryInterfacesInfo, InterfaceInfo{ + ipResult: ¤t.Result{ + IPs: []*current.IPConfig{{Address: *ipnet}}, + }, + }) + } + return ipamAddResult, nil } diff --git a/cni/network/multitenancy_test.go b/cni/network/multitenancy_test.go index e3a2620af1..fe9e3da127 100644 --- a/cni/network/multitenancy_test.go +++ b/cni/network/multitenancy_test.go @@ -287,7 +287,7 @@ func TestCleanupMultitenancyResources(t *testing.T) { }, infraIPNet: &cniTypesCurr.Result{}, plugin: &NetPlugin{ - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), }, }, expected: args{ @@ -298,7 +298,7 @@ func TestCleanupMultitenancyResources(t *testing.T) { }, infraIPNet: &cniTypesCurr.Result{}, plugin: &NetPlugin{ - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), }, }, }, @@ -413,7 +413,7 @@ func TestGetMultiTenancyCNIResult(t *testing.T) { IPAM: cni.IPAM{Type: "azure-vnet-ipam"}, }, plugin: &NetPlugin{ - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), multitenancyClient: &Multitenancy{ netioshim: &mockNetIOShim{}, cnsclient: &MockCNSClient{ @@ -626,7 +626,7 @@ func TestGetMultiTenancyCNIResultUnsupportedAPI(t *testing.T) { IPAM: cni.IPAM{Type: "azure-vnet-ipam"}, }, plugin: &NetPlugin{ - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), multitenancyClient: &Multitenancy{ netioshim: &mockNetIOShim{}, cnsclient: &MockCNSClient{ @@ -765,7 +765,7 @@ func TestGetMultiTenancyCNIResultNotFound(t *testing.T) { IPAM: cni.IPAM{Type: "azure-vnet-ipam"}, }, plugin: &NetPlugin{ - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), multitenancyClient: &Multitenancy{ netioshim: &mockNetIOShim{}, cnsclient: &MockCNSClient{ @@ -800,7 +800,7 @@ func TestGetMultiTenancyCNIResultNotFound(t *testing.T) { IPAM: cni.IPAM{Type: "azure-vnet-ipam"}, }, plugin: &NetPlugin{ - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), multitenancyClient: &Multitenancy{ netioshim: &mockNetIOShim{}, cnsclient: &MockCNSClient{ diff --git a/cni/network/network_test.go b/cni/network/network_test.go index 9d72757557..c53a61eabe 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -77,7 +77,7 @@ func GetTestResources() *NetPlugin { plugin.report = &telemetry.CNIReport{} mockNetworkManager := acnnetwork.NewMockNetworkmanager() plugin.nm = mockNetworkManager - plugin.ipamInvoker = NewMockIpamInvoker(isIPv6, false, false) + plugin.ipamInvoker = NewMockIpamInvoker(isIPv6, false, false, false, false) return plugin } @@ -375,9 +375,9 @@ func TestIpamAddFail(t *testing.T) { for i, method := range tt.methods { fmt.Println("method", method, "wanterr", tt.wantErr[i]) if tt.wantErr[i] { - plugin.ipamInvoker = NewMockIpamInvoker(false, true, false) + plugin.ipamInvoker = NewMockIpamInvoker(false, true, false, false, false) } else { - plugin.ipamInvoker = NewMockIpamInvoker(false, false, false) + plugin.ipamInvoker = NewMockIpamInvoker(false, false, false, false, false) } if method == CNI_ADD { @@ -430,7 +430,7 @@ func TestIpamDeleteFail(t *testing.T) { err := plugin.Add(tt.args) require.NoError(t, err) - plugin.ipamInvoker = NewMockIpamInvoker(false, true, false) + plugin.ipamInvoker = NewMockIpamInvoker(false, true, false, false, false) err = plugin.Delete(args) if tt.wantErr { require.Error(t, err) @@ -460,7 +460,7 @@ func TestAddDualStack(t *testing.T) { plugin: &NetPlugin{ Plugin: cniPlugin, nm: acnnetwork.NewMockNetworkmanager(), - ipamInvoker: NewMockIpamInvoker(true, false, false), + ipamInvoker: NewMockIpamInvoker(true, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, }, @@ -471,7 +471,7 @@ func TestAddDualStack(t *testing.T) { plugin: &NetPlugin{ Plugin: cniPlugin, nm: acnnetwork.NewMockNetworkmanager(), - ipamInvoker: NewMockIpamInvoker(true, false, true), + ipamInvoker: NewMockIpamInvoker(true, false, true, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, }, @@ -517,7 +517,7 @@ func TestPluginGet(t *testing.T) { plugin: &NetPlugin{ Plugin: plugin, nm: acnnetwork.NewMockNetworkmanager(), - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, }, @@ -529,7 +529,7 @@ func TestPluginGet(t *testing.T) { plugin: &NetPlugin{ Plugin: plugin, nm: acnnetwork.NewMockNetworkmanager(), - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, }, @@ -542,7 +542,7 @@ func TestPluginGet(t *testing.T) { plugin: &NetPlugin{ Plugin: plugin, nm: acnnetwork.NewMockNetworkmanager(), - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, }, @@ -1103,3 +1103,75 @@ func TestGetPodSubnetNatInfo(t *testing.T) { require.Empty(t, natInfo, "linux podsubnet natInfo should be empty") } } + +func TestPluginSwiftV2Add(t *testing.T) { + plugin, _ := cni.NewPlugin("name", "0.3.0") + + localNwCfg := cni.NetworkConfig{ + CNIVersion: "0.3.0", + Name: "swiftv2", + ExecutionMode: string(util.V4Overlay), + EnableExactMatchForPodName: true, + Master: "eth0", + } + + tests := []struct { + name string + plugin *NetPlugin + args *cniSkel.CmdArgs + wantErr bool + wantErrMsg string + }{ + { + name: "SwiftV2 Add Happy path", + plugin: &NetPlugin{ + Plugin: plugin, + nm: acnnetwork.NewMockNetworkmanager(), + ipamInvoker: NewMockIpamInvoker(false, false, false, true, false), + report: &telemetry.CNIReport{}, + tb: &telemetry.TelemetryBuffer{}, + }, + args: &cniSkel.CmdArgs{ + StdinData: localNwCfg.Serialize(), + ContainerID: "test-container", + Netns: "test-container", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + }, + wantErr: false, + }, + { + name: "SwiftV2 Add fail", + plugin: &NetPlugin{ + Plugin: plugin, + nm: acnnetwork.NewMockNetworkmanager(), + ipamInvoker: NewMockIpamInvoker(false, false, false, true, true), + report: &telemetry.CNIReport{}, + tb: &telemetry.TelemetryBuffer{}, + }, + args: &cniSkel.CmdArgs{ + StdinData: localNwCfg.Serialize(), + ContainerID: "test-container", + Netns: "test-container", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.plugin.Add(tt.args) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErrMsg, "Expected %v but got %+v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + endpoints, _ := tt.plugin.nm.GetAllEndpoints(localNwCfg.Name) + require.Condition(t, assert.Comparison(func() bool { return len(endpoints) == 1 })) + } + }) + } +} diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index 3b4e92990b..ed94c528be 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -92,7 +92,7 @@ func TestPluginSecondAddSamePodWindows(t *testing.T) { plugin: &NetPlugin{ Plugin: plugin, nm: network.NewMockNetworkmanager(), - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, }, @@ -111,7 +111,7 @@ func TestPluginSecondAddSamePodWindows(t *testing.T) { plugin: &NetPlugin{ Plugin: plugin, nm: network.NewMockNetworkmanager(), - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, }, @@ -351,7 +351,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { plugin: &NetPlugin{ Plugin: plugin, nm: network.NewMockNetworkmanager(), - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, }, @@ -386,7 +386,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { plugin: &NetPlugin{ Plugin: plugin, nm: network.NewMockNetworkmanager(), - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, }, @@ -421,7 +421,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { plugin: &NetPlugin{ Plugin: plugin, nm: network.NewMockNetworkmanager(), - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, }, @@ -456,7 +456,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { plugin: &NetPlugin{ Plugin: plugin, nm: network.NewMockNetworkmanager(), - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, }, @@ -491,7 +491,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { plugin: &NetPlugin{ Plugin: plugin, nm: network.NewMockNetworkmanager(), - ipamInvoker: NewMockIpamInvoker(false, false, false), + ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, }, From 73b0a9d006565e35667eb96b1381c5a67fdd52ea Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 5 Oct 2023 15:25:55 +0000 Subject: [PATCH 20/64] address nit comments --- cni/network/invoker_cns.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 909fea82b0..6d9ce5ec5d 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -21,6 +21,10 @@ import ( "go.uber.org/zap" ) +const ( + ExpectedNumInterfacesWithDefaultRoutes = 1 +) + var ( errEmptyCNIArgs = errors.New("empty CNI cmd args not allowed") errInvalidArgs = errors.New("invalid arg(s)") @@ -142,7 +146,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } logger.Info("Received info for pod", - zap.Any("ipinfo", info), + zap.Any("ipInfo", info), zap.Any("podInfo", podInfo)) //nolint:exhaustive // ignore exhaustive types check @@ -169,7 +173,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } // Make sure default routes exist for 1 interface - if numInterfacesWithDefaultRoutes != 1 { + if numInterfacesWithDefaultRoutes != ExpectedNumInterfacesWithDefaultRoutes { return IPAMAddResult{}, errInvalidDefaultRouting } From 26cf642d2e53f3893eaba1696e45772d6dd75027 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 9 Oct 2023 15:30:56 +0000 Subject: [PATCH 21/64] add comments --- cni/network/invoker_azure_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index 8fc27276c6..d000d899c6 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -82,6 +82,7 @@ func getCIDRNotationForAddress(ipaddresswithcidr string) *net.IPNet { return ipnet } +// getSingleResult returns an IPConfig with v4 or v6 IPNet func getSingleResult(ip string) []*cniTypesCurr.Result { return []*cniTypesCurr.Result{ { @@ -94,6 +95,7 @@ func getSingleResult(ip string) []*cniTypesCurr.Result { } } +// getResult will return a slice of IPConfigs func getResult(ips ...string) *cniTypesCurr.Result { res := &cniTypesCurr.Result{} for _, ip := range ips { From ae09219dae9dd514e1729dff5347c3ad29070c80 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 9 Oct 2023 22:06:12 +0000 Subject: [PATCH 22/64] address comments --- cni/network/invoker_cns.go | 1 + cni/network/invoker_mock.go | 1 + cni/network/network_windows_test.go | 14 +++++++------- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 6d9ce5ec5d..abe2bddfe5 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -152,6 +152,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro //nolint:exhaustive // ignore exhaustive types check switch info.nicType { case cns.DelegatedVMNIC: + // only handling single v4 PodIPInfo for DelegatedVMNICs at the moment, will have to update once v6 gets added if !info.skipDefaultRoutes { numInterfacesWithDefaultRoutes++ } diff --git a/cni/network/invoker_mock.go b/cni/network/invoker_mock.go index 98f081e737..db61751ce2 100644 --- a/cni/network/invoker_mock.go +++ b/cni/network/invoker_mock.go @@ -95,6 +95,7 @@ func (invoker *MockIpamInvoker) Add(opt IPAMAddConfig) (ipamAddResult IPAMAddRes ipResult: ¤t.Result{ IPs: []*current.IPConfig{{Address: *ipnet}}, }, + nicType: cns.DelegatedVMNIC, }) } diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index ed94c528be..2269b19486 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -91,7 +91,7 @@ func TestPluginSecondAddSamePodWindows(t *testing.T) { }, plugin: &NetPlugin{ Plugin: plugin, - nm: network.NewMockNetworkmanager(), + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -110,7 +110,7 @@ func TestPluginSecondAddSamePodWindows(t *testing.T) { }, plugin: &NetPlugin{ Plugin: plugin, - nm: network.NewMockNetworkmanager(), + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -350,7 +350,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { name: "Get Network Name from CNS with correct CIDR", plugin: &NetPlugin{ Plugin: plugin, - nm: network.NewMockNetworkmanager(), + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -385,7 +385,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { name: "Get Network Name from CNS with malformed CIDR #1", plugin: &NetPlugin{ Plugin: plugin, - nm: network.NewMockNetworkmanager(), + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -420,7 +420,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { name: "Get Network Name from CNS with malformed CIDR #2", plugin: &NetPlugin{ Plugin: plugin, - nm: network.NewMockNetworkmanager(), + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -455,7 +455,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { name: "Get Network Name from CNS without NetNS", plugin: &NetPlugin{ Plugin: plugin, - nm: network.NewMockNetworkmanager(), + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -490,7 +490,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { name: "Get Network Name from CNS without multitenancy", plugin: &NetPlugin{ Plugin: plugin, - nm: network.NewMockNetworkmanager(), + nm: network.NewMockNetworkmanager(network.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, From af30d9a6bc139dc2168f8514a462ca9952e2dbbd Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 11 Oct 2023 15:08:47 +0000 Subject: [PATCH 23/64] fix casing --- cni/network/invoker_cns.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index abe2bddfe5..c1a2995bf1 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -22,7 +22,7 @@ import ( ) const ( - ExpectedNumInterfacesWithDefaultRoutes = 1 + expectedNumInterfacesWithDefaultRoutes = 1 ) var ( @@ -174,7 +174,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } // Make sure default routes exist for 1 interface - if numInterfacesWithDefaultRoutes != ExpectedNumInterfacesWithDefaultRoutes { + if numInterfacesWithDefaultRoutes != expectedNumInterfacesWithDefaultRoutes { return IPAMAddResult{}, errInvalidDefaultRouting } From b09b2ca8a15a15bb4fa3e8cb85e743ff7187ff81 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 11 Oct 2023 19:13:26 +0000 Subject: [PATCH 24/64] address comments --- cni/network/invoker_cns.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index c1a2995bf1..674c37beb5 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -162,8 +162,11 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro } default: // only count dualstack interface once - if addResult.defaultInterfaceInfo.ipResult == nil && !info.skipDefaultRoutes { - numInterfacesWithDefaultRoutes++ + if addResult.defaultInterfaceInfo.ipResult == nil { + addResult.defaultInterfaceInfo.ipResult = &cniTypesCurr.Result{} + if !info.skipDefaultRoutes { + numInterfacesWithDefaultRoutes++ + } } overlayMode := (invoker.ipamMode == util.V4Overlay) || (invoker.ipamMode == util.DualStackOverlay) || (invoker.ipamMode == util.Overlay) @@ -368,10 +371,6 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add if ip := net.ParseIP(info.podIPAddress); ip != nil { defaultInterfaceInfo := addResult.defaultInterfaceInfo.ipResult - if defaultInterfaceInfo == nil { - defaultInterfaceInfo = &cniTypesCurr.Result{} - } - defaultRouteDstPrefix := network.Ipv4DefaultRouteDstPrefix if ip.To4() == nil { defaultRouteDstPrefix = network.Ipv6DefaultRouteDstPrefix From cd97880f1c2b2b23a4c01b1e9e9ae4cc2646075b Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 8 Sep 2023 18:02:44 +0000 Subject: [PATCH 25/64] feat: update invokers to support swift 2 --- cni/network/invoker_azure_test.go | 14 ++------- cni/network/invoker_cns_test.go | 48 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index d000d899c6..58346d061f 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -35,23 +35,13 @@ func (d *add) DelegateAdd(pluginName string, nwCfg *cni.NetworkConfig) (*cniType if d.errv6 != nil { return nil, d.errv6 } - if d.resultsIPv6 == nil || d.resultsIPv6Index-1 > len(d.resultsIPv6) { - return nil, errors.New("no more ipv6 results in mock available") //nolint:goerr113 - } - res := d.resultsIPv6[d.resultsIPv6Index] - d.resultsIPv6Index++ - return res, nil + return d.resultsIPv6, nil } if d.errv4 != nil { return nil, d.errv4 } - if d.resultsIPv4 == nil || d.resultsIPv4Index-1 > len(d.resultsIPv4) { - return nil, errors.New("no more ipv4 results in mock available") //nolint:goerr113 - } - res := d.resultsIPv4[d.resultsIPv4Index] - d.resultsIPv4Index++ - return res, nil + return d.resultsIPv4, nil } type del struct { diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 0b569e08ac..164805ba79 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -68,12 +68,21 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { } tests := []struct { +<<<<<<< HEAD name string fields fields args args wantDefaultResult *cniTypesCurr.Result wantSecondaryInterfacesInfo InterfaceInfo wantErr bool +======= + name string + fields fields + args args + wantDefaultResult *cniTypesCurr.Result + wantMultitenantResult *cniTypesCurr.Result + wantErr bool +>>>>>>> 5267b700 (feat: update invokers to support swift 2) }{ { name: "Test happy CNI Overlay add in v4overlay ipamMode", @@ -273,16 +282,25 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, +<<<<<<< HEAD NICType: cns.InfraNIC, SkipDefaultRoutes: true, +======= + AddressType: cns.Default, +>>>>>>> 5267b700 (feat: update invokers to support swift 2) }, { PodIPConfig: cns.IPSubnet{ IPAddress: "20.240.1.242", PrefixLength: 24, }, +<<<<<<< HEAD NICType: cns.DelegatedVMNIC, MacAddress: macAddress, +======= + AddressType: cns.Secondary, + MacAddress: "12:34:56:78:9a:bc", +>>>>>>> 5267b700 (feat: update invokers to support swift 2) }, }, Response: cns.Response{ @@ -318,6 +336,7 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { }, }, }, +<<<<<<< HEAD wantSecondaryInterfacesInfo: InterfaceInfo{ ipResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ @@ -383,6 +402,17 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { }, }, err: nil, +======= + wantMultitenantResult: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: *getCIDRNotationForAddress("20.240.1.242/24"), + }, + }, + Interfaces: []*cniTypesCurr.Interface{ + { + Mac: "12:34:56:78:9a:bc", +>>>>>>> 5267b700 (feat: update invokers to support swift 2) }, }, }, @@ -484,10 +514,17 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { require.NoError(err) } +<<<<<<< HEAD fmt.Printf("want:%+v\nrest:%+v\n", tt.wantSecondaryInterfacesInfo, ipamAddResult.secondaryInterfacesInfo) require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultInterfaceInfo.ipResult, "incorrect default response") if tt.wantSecondaryInterfacesInfo.ipResult != nil { require.EqualValues(tt.wantSecondaryInterfacesInfo, ipamAddResult.secondaryInterfacesInfo[0], "incorrect multitenant response") +======= + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.cniResults) + require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultCniResult.ipResult, "incorrect default response") + if tt.wantMultitenantResult != nil { + require.Equalf(tt.wantMultitenantResult, ipamAddResult.cniResults[0].ipResult, "incorrect multitenant response") +>>>>>>> 5267b700 (feat: update invokers to support swift 2) } }) } @@ -710,10 +747,17 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { require.NoError(err) } +<<<<<<< HEAD fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.secondaryInterfacesInfo) require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultInterfaceInfo.ipResult, "incorrect default response") if tt.wantMultitenantResult != nil { require.Equalf(tt.wantMultitenantResult, ipamAddResult.secondaryInterfacesInfo[0].ipResult, "incorrect multitenant response") +======= + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.cniResults) + require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultCniResult.ipResult, "incorrect default response") + if tt.wantMultitenantResult != nil { + require.Equalf(tt.wantMultitenantResult, ipamAddResult.cniResults[0].ipResult, "incorrect multitenant response") +>>>>>>> 5267b700 (feat: update invokers to support swift 2) } }) } @@ -829,7 +873,11 @@ func TestCNSIPAMInvoker_Add_UnsupportedAPI(t *testing.T) { t.Fatalf("expected an error %+v but none received", err) } require.NoError(err) +<<<<<<< HEAD require.Equalf(tt.want, ipamAddResult.defaultInterfaceInfo.ipResult, "incorrect ipv4 response") +======= + require.Equalf(tt.want, ipamAddResult.defaultCniResult.ipResult, "incorrect ipv4 response") +>>>>>>> 5267b700 (feat: update invokers to support swift 2) }) } } From fbb1fb67edb0a9918cc8beb43f39b559b496132f Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Tue, 3 Oct 2023 17:02:31 +0000 Subject: [PATCH 26/64] address comments --- cni/network/invoker_azure_test.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index 58346d061f..d000d899c6 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -35,13 +35,23 @@ func (d *add) DelegateAdd(pluginName string, nwCfg *cni.NetworkConfig) (*cniType if d.errv6 != nil { return nil, d.errv6 } - return d.resultsIPv6, nil + if d.resultsIPv6 == nil || d.resultsIPv6Index-1 > len(d.resultsIPv6) { + return nil, errors.New("no more ipv6 results in mock available") //nolint:goerr113 + } + res := d.resultsIPv6[d.resultsIPv6Index] + d.resultsIPv6Index++ + return res, nil } if d.errv4 != nil { return nil, d.errv4 } - return d.resultsIPv4, nil + if d.resultsIPv4 == nil || d.resultsIPv4Index-1 > len(d.resultsIPv4) { + return nil, errors.New("no more ipv4 results in mock available") //nolint:goerr113 + } + res := d.resultsIPv4[d.resultsIPv4Index] + d.resultsIPv4Index++ + return res, nil } type del struct { From b45906d5538e4b949f390d358af71711d99820fa Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 8 Sep 2023 18:02:44 +0000 Subject: [PATCH 27/64] feat: update invokers to support swift 2 --- cni/network/invoker_cns.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 674c37beb5..4123977d56 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -181,6 +181,8 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro return IPAMAddResult{}, errInvalidDefaultRouting } + addResult.defaultCniResult.isDefaultInterface = !isDefaultInterfaceSet + return addResult, nil } From 053dfe6e33f330ca84d5662d848c0724c9c89dbf Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 8 Sep 2023 18:02:44 +0000 Subject: [PATCH 28/64] feat: update invokers to support swift 2 --- cni/network/invoker_cns.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 4123977d56..674c37beb5 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -181,8 +181,6 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro return IPAMAddResult{}, errInvalidDefaultRouting } - addResult.defaultCniResult.isDefaultInterface = !isDefaultInterfaceSet - return addResult, nil } From 20db7476edcb9997994ab68194f1864ddaeb5e36 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 8 Sep 2023 18:12:01 +0000 Subject: [PATCH 29/64] feat: update endpoint clients for swift 2 --- netio/mocknetio.go | 23 +- netio/netio.go | 18 ++ network/secondary_endpoint_client_linux.go | 152 ++++++++++++ network/secondary_endpoint_linux_test.go | 258 ++++++++++++++++++++ network/transparent_endpoint_linux_test.go | 9 +- network/transparent_endpointclient_linux.go | 106 ++++---- 6 files changed, 509 insertions(+), 57 deletions(-) create mode 100644 network/secondary_endpoint_client_linux.go create mode 100644 network/secondary_endpoint_linux_test.go diff --git a/netio/mocknetio.go b/netio/mocknetio.go index b002e4dde4..b80a44bbb1 100644 --- a/netio/mocknetio.go +++ b/netio/mocknetio.go @@ -1,6 +1,7 @@ package netio import ( + "bytes" "errors" "fmt" "net" @@ -16,7 +17,10 @@ type MockNetIO struct { } // ErrMockNetIOFail - mock netio error -var ErrMockNetIOFail = errors.New("netio fail") +var ( + ErrMockNetIOFail = errors.New("netio fail") + hwAddr, _ = net.ParseMAC("ab:cd:ef:12:34:56") +) func NewMockNetIO(fail bool, failAttempt int) *MockNetIO { return &MockNetIO{ @@ -40,8 +44,6 @@ func (netshim *MockNetIO) GetNetworkInterfaceByName(name string) (*net.Interface return netshim.getInterfaceFn(name) } - hwAddr, _ := net.ParseMAC("ab:cd:ef:12:34:56") - return &net.Interface{ //nolint:gomnd // Dummy MTU MTU: 1000, @@ -55,3 +57,18 @@ func (netshim *MockNetIO) GetNetworkInterfaceByName(name string) (*net.Interface func (netshim *MockNetIO) GetNetworkInterfaceAddrs(iface *net.Interface) ([]net.Addr, error) { return []net.Addr{}, nil } + +func (netshim *MockNetIO) GetNetworkInterfaceByMac(mac net.HardwareAddr) (*net.Interface, error) { + if !bytes.Equal(mac, hwAddr) { + return nil, fmt.Errorf("%w: %s", ErrMockNetIOFail, mac) + } + + return &net.Interface{ + //nolint:gomnd // Dummy MTU + MTU: 1000, + Name: "eth1", + HardwareAddr: mac, + //nolint:gomnd // Dummy interface index + Index: 2, + }, nil +} diff --git a/netio/netio.go b/netio/netio.go index 0f2bf03274..9462c8ed33 100644 --- a/netio/netio.go +++ b/netio/netio.go @@ -1,6 +1,7 @@ package netio import ( + "bytes" "net" "github.com/pkg/errors" @@ -10,10 +11,12 @@ import ( type NetIOInterface interface { GetNetworkInterfaceByName(name string) (*net.Interface, error) GetNetworkInterfaceAddrs(iface *net.Interface) ([]net.Addr, error) + GetNetworkInterfaceByMac(mac net.HardwareAddr) (*net.Interface, error) } // ErrInterfaceNil - errors out when interface is nil var ErrInterfaceNil = errors.New("Interface is nil") +var ErrInterfaceNotFound = errors.New("Inteface not found") type NetIO struct{} @@ -30,3 +33,18 @@ func (ns *NetIO) GetNetworkInterfaceAddrs(iface *net.Interface) ([]net.Addr, err addrs, err := iface.Addrs() return addrs, errors.Wrap(err, "GetNetworkInterfaceAddrs failed") } + +func (ns *NetIO) GetNetworkInterfaceByMac(mac net.HardwareAddr) (*net.Interface, error) { + ifaces, err := net.Interfaces() + if err != nil { + return nil, errors.Wrap(err, "GetNetworkInterfaceByMac failed") + } + + for _, iface := range ifaces { + if bytes.Equal(iface.HardwareAddr, mac) { + return &iface, nil + } + } + + return nil, ErrInterfaceNotFound +} diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go new file mode 100644 index 0000000000..ba85bb7455 --- /dev/null +++ b/network/secondary_endpoint_client_linux.go @@ -0,0 +1,152 @@ +package network + +import ( + "net" + + "github.com/pkg/errors" + + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netio" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/platform" +) + +var errorSecondaryEndpointClient = errors.New("SecondaryEndpointClient Error") + +func newErrorSecondaryEndpointClient(err error) error { + return errors.Wrapf(err, "%s", errorSecondaryEndpointClient) +} + +type SecondaryEndpointClient struct { + netlink netlink.NetlinkInterface + netioshim netio.NetIOInterface + plClient platform.ExecClient + netUtilsClient networkutils.NetworkUtils + ep *endpoint +} + +func NewSecondaryEndpointClient( + nl netlink.NetlinkInterface, + plc platform.ExecClient, + endpoint *endpoint, +) *SecondaryEndpointClient { + client := &SecondaryEndpointClient{ + netlink: nl, + netioshim: &netio.NetIO{}, + plClient: plc, + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + ep: endpoint, + } + + return client +} + +func (client *SecondaryEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { + iface, err := client.netioshim.GetNetworkInterfaceByMac(epInfo.MacAddress) + if err != nil { + return newErrorSecondaryEndpointClient(err) + } + + epInfo.IfName = iface.Name + if _, exists := client.ep.SecondaryInterfaces[iface.Name]; exists { + return newErrorSecondaryEndpointClient(errors.New(iface.Name + " already exists")) + } + client.ep.SecondaryInterfaces[iface.Name] = &InterfaceInfo{ + Name: iface.Name, + MacAddress: epInfo.MacAddress, + IPAddress: epInfo.IPAddresses, + AddressType: epInfo.AddressType, + IsDefaultInterface: epInfo.IsDefaultInterface, + } + + return nil +} + +func (client *SecondaryEndpointClient) AddEndpointRules(_ *EndpointInfo) error { + return nil +} + +func (client *SecondaryEndpointClient) DeleteEndpointRules(ep *endpoint) { + // ip route del dev + // Deleting the route set up for routing the incoming packets to pod + for ifName, ifInfo := range ep.SecondaryInterfaces { + log.Printf("[net] deleting routes on %s: %+v", ifName, ifInfo.Routes) + if err := deleteRoutes(client.netlink, client.netioshim, ifName, ifInfo.Routes); err != nil { + log.Printf("[net] Failed to delete routes %+v on %s: %v", ifInfo.Routes, ifName, err) + } + } +} + +func (client *SecondaryEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { + // Move the container interface to container's network namespace. + log.Printf("[net] Setting link %v netns %v.", epInfo.IfName, epInfo.NetNsPath) + if err := client.netlink.SetLinkNetNs(epInfo.IfName, nsID); err != nil { + return newErrorSecondaryEndpointClient(err) + } + + return nil +} + +func (client *SecondaryEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { + log.Printf("[net] Setting link %v state up.", epInfo.IfName) + + return client.netlink.SetLinkState(epInfo.IfName, true) +} + +func (client *SecondaryEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + if err := client.netUtilsClient.AssignIPToInterface(epInfo.IfName, epInfo.IPAddresses); err != nil { + return newErrorSecondaryEndpointClient(err) + } + + ifInfo, exists := client.ep.SecondaryInterfaces[epInfo.IfName] + if !exists { + return newErrorSecondaryEndpointClient(errors.New(epInfo.IfName + " does not exist")) + } + + if epInfo.IsDefaultInterface { + // add route for virtualgwip + // ip route add 169.254.1.1/32 dev ethX + virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) + routeInfo := RouteInfo{ + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + } + if err := addRoutes(client.netlink, client.netioshim, epInfo.IfName, []RouteInfo{routeInfo}); err != nil { + return newErrorSecondaryEndpointClient(err) + } + + // TO-DO: will default routes need to be cleaned up even though they're added in pod netns? + // looks like interface goes back to default state (down without routes) after deleting pod + ifInfo.Routes = append(ifInfo.Routes, routeInfo) + + // ip route add default via 169.254.1.1 dev ethX + _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) + dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} + routeInfo = RouteInfo{ + Dst: dstIP, + Gw: virtualGwIP, + } + if err := addRoutes(client.netlink, client.netioshim, epInfo.IfName, []RouteInfo{routeInfo}); err != nil { + return newErrorSecondaryEndpointClient(err) + } + + // TO-DO: will default routes need to be cleaned up even though they're added in pod netns? + // looks like interface goes back to default state (down without routes) after deleting pod + ifInfo.Routes = append(ifInfo.Routes, routeInfo) + } + + if err := addRoutes(client.netlink, client.netioshim, epInfo.IfName, epInfo.Routes); err != nil { + return newErrorSecondaryEndpointClient(err) + } + + ifInfo.Routes = append(ifInfo.Routes, epInfo.Routes...) + + return nil +} + +func (client *SecondaryEndpointClient) DeleteEndpoints(_ *endpoint) error { + // TO-DO: try to clean up and move back to default ns? + // looks like interface goes back to default state (down without routes) after deleting pod + return nil +} diff --git a/network/secondary_endpoint_linux_test.go b/network/secondary_endpoint_linux_test.go new file mode 100644 index 0000000000..88f823d254 --- /dev/null +++ b/network/secondary_endpoint_linux_test.go @@ -0,0 +1,258 @@ +//go:build linux +// +build linux + +package network + +import ( + "net" + "testing" + + "github.com/Azure/azure-container-networking/netio" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/networkutils" + "github.com/Azure/azure-container-networking/platform" + "github.com/stretchr/testify/require" +) + +func TestSecondaryAddEndpoints(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + mac, _ := net.ParseMAC("ab:cd:ef:12:34:56") + invalidMac, _ := net.ParseMAC("12:34:56:ab:cd:ef") + + tests := []struct { + name string + client *SecondaryEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Add endpoints", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + ep: &endpoint{SecondaryInterfaces: make(map[string]*InterfaceInfo)}, + }, + epInfo: &EndpointInfo{MacAddress: mac}, + wantErr: false, + }, + { + name: "Add endpoints invalid mac", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + ep: &endpoint{SecondaryInterfaces: make(map[string]*InterfaceInfo)}, + }, + epInfo: &EndpointInfo{MacAddress: invalidMac}, + wantErr: true, + wantErrMsg: "SecondaryEndpointClient Error: " + netio.ErrMockNetIOFail.Error() + ": " + invalidMac.String(), + }, + { + name: "Add endpoints interface already added", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + ep: &endpoint{SecondaryInterfaces: map[string]*InterfaceInfo{"eth1": {Name: "eth1"}}}, + }, + epInfo: &EndpointInfo{MacAddress: mac}, + wantErr: true, + wantErrMsg: "SecondaryEndpointClient Error: eth1 already exists", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.AddEndpoints(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + require.Equal(t, tt.client.ep.SecondaryInterfaces["eth1"].MacAddress, tt.epInfo.MacAddress) + } + }) + } +} + +func TestSecondaryDeleteEndpointsRules(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + tests := []struct { + name string + client *SecondaryEndpointClient + ep *endpoint + }{ + { + name: "Delete endpoint rules happy path", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + }, + ep: &endpoint{ + SecondaryInterfaces: map[string]*InterfaceInfo{ + "eth1": { + Name: "eth1", + Routes: []RouteInfo{ + { + Dst: net.IPNet{IP: net.ParseIP("192.168.0.4"), Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)}, + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + tt.client.DeleteEndpointRules(tt.ep) + }) + } +} + +func TestSecondaryConfigureContainerInterfacesAndRoutes(t *testing.T) { + nl := netlink.NewMockNetlink(false, "") + plc := platform.NewMockExecClient(false) + + tests := []struct { + name string + client *SecondaryEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + { + name: "Configure Interface and routes happy path", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + ep: &endpoint{SecondaryInterfaces: map[string]*InterfaceInfo{"eth1": {Name: "eth1"}}}, + }, + epInfo: &EndpointInfo{ + IfName: "eth1", + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + IsDefaultInterface: true, + }, + wantErr: false, + }, + { + name: "Configure Interface and routes assign ip fail", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + ep: &endpoint{SecondaryInterfaces: map[string]*InterfaceInfo{"eth1": {Name: "eth1"}}}, + }, + epInfo: &EndpointInfo{ + IfName: "eth1", + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + IsDefaultInterface: true, + }, + wantErr: true, + wantErrMsg: "netlink fail", + }, + { + name: "Configure Interface and routes add routes fail", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 2), + ep: &endpoint{SecondaryInterfaces: map[string]*InterfaceInfo{"eth1": {Name: "eth1"}}}, + }, + epInfo: &EndpointInfo{ + IfName: "eth1", + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + IsDefaultInterface: true, + }, + wantErr: true, + wantErrMsg: netio.ErrMockNetIOFail.Error(), + }, + { + name: "Configure Interface and routes add routes invalid interface name", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + ep: &endpoint{SecondaryInterfaces: map[string]*InterfaceInfo{"eth1": {Name: "eth1"}}}, + }, + epInfo: &EndpointInfo{ + IfName: "eth2", + IPAddresses: []net.IPNet{ + { + IP: net.ParseIP("192.168.0.4"), + Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), + }, + }, + IsDefaultInterface: true, + }, + wantErr: true, + wantErrMsg: "SecondaryEndpointClient Error: eth2 does not exist", + }, + { + name: "Configure Interface and routes add routes fail on non default interface", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(true, 2), + ep: &endpoint{SecondaryInterfaces: map[string]*InterfaceInfo{"eth1": {Name: "eth1"}}}, + }, + epInfo: &EndpointInfo{ + IfName: "eth1", + Routes: []RouteInfo{ + { + Dst: net.IPNet{IP: net.ParseIP("192.168.0.4"), Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)}, + }, + }, + IsDefaultInterface: true, + }, + wantErr: true, + wantErrMsg: netio.ErrMockNetIOFail.Error(), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.ConfigureContainerInterfacesAndRoutes(tt.epInfo) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/network/transparent_endpoint_linux_test.go b/network/transparent_endpoint_linux_test.go index 4fa69772f1..e140dbebb5 100644 --- a/network/transparent_endpoint_linux_test.go +++ b/network/transparent_endpoint_linux_test.go @@ -57,7 +57,7 @@ func TestTransAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "TransparentEndpointClient Error : " + netlink.ErrorMockNetlink.Error() + " : netlink fail", + wantErrMsg: "TransparentEndpointClient Error: " + netlink.ErrorMockNetlink.Error() + " : netlink fail", }, { name: "Add endpoints get interface fail for old veth", @@ -86,7 +86,7 @@ func TestTransAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "TransparentEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":eth0", + wantErrMsg: "TransparentEndpointClient Error: " + netio.ErrMockNetIOFail.Error() + ":eth0", }, { name: "Add endpoints get interface fail for host veth", @@ -101,7 +101,7 @@ func TestTransAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "TransparentEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":azvcontainer", + wantErrMsg: "TransparentEndpointClient Error: " + netio.ErrMockNetIOFail.Error() + ":azvcontainer", }, { name: "get interface fail for container veth", @@ -116,7 +116,7 @@ func TestTransAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "TransparentEndpointClient Error : " + netio.ErrMockNetIOFail.Error() + ":azvhost", + wantErrMsg: "TransparentEndpointClient Error: " + netio.ErrMockNetIOFail.Error() + ":azvhost", }, } @@ -370,6 +370,7 @@ func TestTransConfigureContainerInterfacesAndRoutes(t *testing.T) { Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), }, }, + IsDefaultInterface: true, }, wantErr: true, wantErrMsg: netio.ErrMockNetIOFail.Error(), diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 884ef9ff30..41687be706 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -1,7 +1,6 @@ package network import ( - "errors" "fmt" "net" @@ -9,6 +8,7 @@ import ( "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" + "github.com/pkg/errors" "go.uber.org/zap" ) @@ -27,8 +27,8 @@ const ( var errorTransparentEndpointClient = errors.New("TransparentEndpointClient Error") -func newErrorTransparentEndpointClient(errStr string) error { - return fmt.Errorf("%w : %s", errorTransparentEndpointClient, errStr) +func newErrorTransparentEndpointClient(err error) error { + return errors.Wrapf(err, "%s", errorTransparentEndpointClient) } type TransparentEndpointClient struct { @@ -81,13 +81,13 @@ func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) erro logger.Info("Deleting old host veth", zap.String("hostVethName", client.hostVethName)) if err = client.netlink.DeleteLink(client.hostVethName); err != nil { logger.Error("Failed to delete old", zap.String("hostVethName", client.hostVethName), zap.Error(err)) - return newErrorTransparentEndpointClient(err.Error()) + return newErrorTransparentEndpointClient(err) } } primaryIf, err := client.netioshim.GetNetworkInterfaceByName(client.hostPrimaryIfName) if err != nil { - return newErrorTransparentEndpointClient(err.Error()) + return newErrorTransparentEndpointClient(err) } mac, err := net.ParseMAC(defaultHostVethHwAddr) @@ -96,7 +96,7 @@ func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) erro } if err = client.netUtilsClient.CreateEndpoint(client.hostVethName, client.containerVethName, mac); err != nil { - return newErrorTransparentEndpointClient(err.Error()) + return newErrorTransparentEndpointClient(err) } defer func() { @@ -109,14 +109,14 @@ func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) erro containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) if err != nil { - return newErrorTransparentEndpointClient(err.Error()) + return newErrorTransparentEndpointClient(err) } client.containerMac = containerIf.HardwareAddr hostVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName) if err != nil { - return newErrorTransparentEndpointClient(err.Error()) + return newErrorTransparentEndpointClient(err) } client.hostVethMac = hostVethIf.HardwareAddr @@ -157,7 +157,7 @@ func (client *TransparentEndpointClient) AddEndpointRules(epInfo *EndpointInfo) } if err := addRoutes(client.netlink, client.netioshim, client.hostVethName, routeInfoList); err != nil { - return newErrorTransparentEndpointClient(err.Error()) + return newErrorTransparentEndpointClient(err) } logger.Info("calling setArpProxy for", zap.String("hostVethName", client.hostVethName)) @@ -196,7 +196,7 @@ func (client *TransparentEndpointClient) MoveEndpointsToContainerNS(epInfo *Endp // Move the container interface to container's network namespace. logger.Info("Setting link netns", zap.String("containerVethName", client.containerVethName), zap.String("NetNsPath", epInfo.NetNsPath)) if err := client.netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil { - return newErrorTransparentEndpointClient(err.Error()) + return newErrorTransparentEndpointClient(err) } return nil @@ -214,7 +214,7 @@ func (client *TransparentEndpointClient) SetupContainerInterfaces(epInfo *Endpoi func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { if err := client.netUtilsClient.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { - return newErrorTransparentEndpointClient(err.Error()) + return newErrorTransparentEndpointClient(err) } // ip route del 10.240.0.0/12 dev eth0 (removing kernel subnet route added by above call) @@ -226,55 +226,61 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e Protocol: netlink.RTPROT_KERNEL, } if err := deleteRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { - return newErrorTransparentEndpointClient(err.Error()) + return newErrorTransparentEndpointClient(err) } } - // add route for virtualgwip - // ip route add 169.254.1.1/32 dev eth0 - virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) - routeInfo := RouteInfo{ - Dst: *virtualGwNet, - Scope: netlink.RT_SCOPE_LINK, - } - if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { - return newErrorTransparentEndpointClient(err.Error()) - } + if epInfo.IsDefaultInterface { + // add route for virtualgwip + // ip route add 169.254.1.1/32 dev eth0 + virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) + routeInfo := RouteInfo{ + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + } + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + return newErrorTransparentEndpointClient(err) + } - // ip route add default via 169.254.1.1 dev eth0 - _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) - dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} - routeInfo = RouteInfo{ - Dst: dstIP, - Gw: virtualGwIP, - } - if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { - return err - } + // ip route add default via 169.254.1.1 dev eth0 + _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) + dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} + routeInfo = RouteInfo{ + Dst: dstIP, + Gw: virtualGwIP, + } + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + return err + } - // arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac - logger.Info("Adding static arp for IP address and MAC in Container namespace", - zap.String("address", virtualGwNet.String()), zap.Any("hostVethMac", client.hostVethMac)) - linkInfo := netlink.LinkInfo{ - Name: client.containerVethName, - IPAddr: virtualGwNet.IP, - MacAddress: client.hostVethMac, - } + // arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac + logger.Info("Adding static arp for IP address and MAC in Container namespace", + zap.String("address", virtualGwNet.String()), zap.Any("hostVethMac", client.hostVethMac)) + linkInfo := netlink.LinkInfo{ + Name: client.containerVethName, + IPAddr: virtualGwNet.IP, + MacAddress: client.hostVethMac, + } - if err := client.netlink.SetOrRemoveLinkAddress(linkInfo, netlink.ADD, netlink.NUD_PROBE); err != nil { - return fmt.Errorf("Adding arp in container failed: %w", err) - } + if err := client.netlink.SetOrRemoveLinkAddress(linkInfo, netlink.ADD, netlink.NUD_PROBE); err != nil { + return fmt.Errorf("Adding arp in container failed: %w", err) + } - // IPv6Mode can be ipv6NAT or dual stack overlay - // set epInfo ipv6Mode to 'dualStackOverlay' to set ipv6Routes and ipv6NeighborEntries for Linux pod in dualStackOverlay ipam mode - if epInfo.IPV6Mode != "" { - if err := client.setupIPV6Routes(); err != nil { - return err + // IPv6Mode can be ipv6NAT or dual stack overlay + // set epInfo ipv6Mode to 'dualStackOverlay' to set ipv6Routes and ipv6NeighborEntries for Linux pod in dualStackOverlay ipam mode + if epInfo.IPV6Mode != "" { + if err := client.setupIPV6Routes(); err != nil { + return err + } + } + + if epInfo.IPV6Mode != "" { + return client.setIPV6NeighEntry() } } - if epInfo.IPV6Mode != "" { - return client.setIPV6NeighEntry() + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, epInfo.Routes); err != nil { + return newErrorTransparentEndpointClient(err) } return nil From abbd281285fbca3052d496bbcf8067a2d24a5082 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 22 Sep 2023 20:54:36 +0000 Subject: [PATCH 30/64] address comments --- network/secondary_endpoint_client_linux.go | 39 +++++---------------- network/transparent_endpointclient_linux.go | 2 ++ 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index ba85bb7455..3c2780b8f2 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -56,7 +56,7 @@ func (client *SecondaryEndpointClient) AddEndpoints(epInfo *EndpointInfo) error Name: iface.Name, MacAddress: epInfo.MacAddress, IPAddress: epInfo.IPAddresses, - AddressType: epInfo.AddressType, + NICType: epInfo.NICType, IsDefaultInterface: epInfo.IsDefaultInterface, } @@ -68,14 +68,6 @@ func (client *SecondaryEndpointClient) AddEndpointRules(_ *EndpointInfo) error { } func (client *SecondaryEndpointClient) DeleteEndpointRules(ep *endpoint) { - // ip route del dev - // Deleting the route set up for routing the incoming packets to pod - for ifName, ifInfo := range ep.SecondaryInterfaces { - log.Printf("[net] deleting routes on %s: %+v", ifName, ifInfo.Routes) - if err := deleteRoutes(client.netlink, client.netioshim, ifName, ifInfo.Routes); err != nil { - log.Printf("[net] Failed to delete routes %+v on %s: %v", ifInfo.Routes, ifName, err) - } - } } func (client *SecondaryEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { @@ -90,8 +82,11 @@ func (client *SecondaryEndpointClient) MoveEndpointsToContainerNS(epInfo *Endpoi func (client *SecondaryEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { log.Printf("[net] Setting link %v state up.", epInfo.IfName) + if err := client.netlink.SetLinkState(epInfo.IfName, true); err != nil { + return newErrorSecondaryEndpointClient(err) + } - return client.netlink.SetLinkState(epInfo.IfName, true) + return nil } func (client *SecondaryEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { @@ -105,35 +100,19 @@ func (client *SecondaryEndpointClient) ConfigureContainerInterfacesAndRoutes(epI } if epInfo.IsDefaultInterface { - // add route for virtualgwip - // ip route add 169.254.1.1/32 dev ethX - virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) - routeInfo := RouteInfo{ - Dst: *virtualGwNet, - Scope: netlink.RT_SCOPE_LINK, - } - if err := addRoutes(client.netlink, client.netioshim, epInfo.IfName, []RouteInfo{routeInfo}); err != nil { - return newErrorSecondaryEndpointClient(err) - } - - // TO-DO: will default routes need to be cleaned up even though they're added in pod netns? - // looks like interface goes back to default state (down without routes) after deleting pod - ifInfo.Routes = append(ifInfo.Routes, routeInfo) - - // ip route add default via 169.254.1.1 dev ethX + // ip route add default via 0.0.0.0 dev ethX _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} - routeInfo = RouteInfo{ + routeInfo := RouteInfo{ Dst: dstIP, - Gw: virtualGwIP, } if err := addRoutes(client.netlink, client.netioshim, epInfo.IfName, []RouteInfo{routeInfo}); err != nil { return newErrorSecondaryEndpointClient(err) } - // TO-DO: will default routes need to be cleaned up even though they're added in pod netns? - // looks like interface goes back to default state (down without routes) after deleting pod ifInfo.Routes = append(ifInfo.Routes, routeInfo) + + return nil } if err := addRoutes(client.netlink, client.netioshim, epInfo.IfName, epInfo.Routes); err != nil { diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 41687be706..f72365185b 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -277,6 +277,8 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e if epInfo.IPV6Mode != "" { return client.setIPV6NeighEntry() } + + return nil } if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, epInfo.Routes); err != nil { From 3b59f9933a7a260e0f23dac016c16c985943c7b0 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 25 Sep 2023 17:25:21 +0000 Subject: [PATCH 31/64] fix lint errs --- network/secondary_endpoint_client_linux.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index 3c2780b8f2..bf5038df89 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -3,13 +3,12 @@ package network import ( "net" - "github.com/pkg/errors" - "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" + "github.com/pkg/errors" ) var errorSecondaryEndpointClient = errors.New("SecondaryEndpointClient Error") @@ -67,7 +66,7 @@ func (client *SecondaryEndpointClient) AddEndpointRules(_ *EndpointInfo) error { return nil } -func (client *SecondaryEndpointClient) DeleteEndpointRules(ep *endpoint) { +func (client *SecondaryEndpointClient) DeleteEndpointRules(_ *endpoint) { } func (client *SecondaryEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { From 5e844a7b3728bfec3dead2c1792659d62efd9801 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 25 Sep 2023 20:43:46 +0000 Subject: [PATCH 32/64] update endpoint clients based on contract changes --- network/secondary_endpoint_client_linux.go | 28 ++++----------------- network/secondary_endpoint_linux_test.go | 5 ---- network/transparent_endpoint_linux_test.go | 1 - network/transparent_endpointclient_linux.go | 2 +- 4 files changed, 6 insertions(+), 30 deletions(-) diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index bf5038df89..4d0cf66723 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -1,8 +1,6 @@ package network import ( - "net" - "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" @@ -52,11 +50,11 @@ func (client *SecondaryEndpointClient) AddEndpoints(epInfo *EndpointInfo) error return newErrorSecondaryEndpointClient(errors.New(iface.Name + " already exists")) } client.ep.SecondaryInterfaces[iface.Name] = &InterfaceInfo{ - Name: iface.Name, - MacAddress: epInfo.MacAddress, - IPAddress: epInfo.IPAddresses, - NICType: epInfo.NICType, - IsDefaultInterface: epInfo.IsDefaultInterface, + Name: iface.Name, + MacAddress: epInfo.MacAddress, + IPAddress: epInfo.IPAddresses, + NICType: epInfo.NICType, + SkipDefaultRoutes: epInfo.SkipDefaultRoutes, } return nil @@ -98,22 +96,6 @@ func (client *SecondaryEndpointClient) ConfigureContainerInterfacesAndRoutes(epI return newErrorSecondaryEndpointClient(errors.New(epInfo.IfName + " does not exist")) } - if epInfo.IsDefaultInterface { - // ip route add default via 0.0.0.0 dev ethX - _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) - dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} - routeInfo := RouteInfo{ - Dst: dstIP, - } - if err := addRoutes(client.netlink, client.netioshim, epInfo.IfName, []RouteInfo{routeInfo}); err != nil { - return newErrorSecondaryEndpointClient(err) - } - - ifInfo.Routes = append(ifInfo.Routes, routeInfo) - - return nil - } - if err := addRoutes(client.netlink, client.netioshim, epInfo.IfName, epInfo.Routes); err != nil { return newErrorSecondaryEndpointClient(err) } diff --git a/network/secondary_endpoint_linux_test.go b/network/secondary_endpoint_linux_test.go index 88f823d254..17ff2dbce8 100644 --- a/network/secondary_endpoint_linux_test.go +++ b/network/secondary_endpoint_linux_test.go @@ -150,7 +150,6 @@ func TestSecondaryConfigureContainerInterfacesAndRoutes(t *testing.T) { Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), }, }, - IsDefaultInterface: true, }, wantErr: false, }, @@ -171,7 +170,6 @@ func TestSecondaryConfigureContainerInterfacesAndRoutes(t *testing.T) { Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), }, }, - IsDefaultInterface: true, }, wantErr: true, wantErrMsg: "netlink fail", @@ -193,7 +191,6 @@ func TestSecondaryConfigureContainerInterfacesAndRoutes(t *testing.T) { Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), }, }, - IsDefaultInterface: true, }, wantErr: true, wantErrMsg: netio.ErrMockNetIOFail.Error(), @@ -215,7 +212,6 @@ func TestSecondaryConfigureContainerInterfacesAndRoutes(t *testing.T) { Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), }, }, - IsDefaultInterface: true, }, wantErr: true, wantErrMsg: "SecondaryEndpointClient Error: eth2 does not exist", @@ -236,7 +232,6 @@ func TestSecondaryConfigureContainerInterfacesAndRoutes(t *testing.T) { Dst: net.IPNet{IP: net.ParseIP("192.168.0.4"), Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)}, }, }, - IsDefaultInterface: true, }, wantErr: true, wantErrMsg: netio.ErrMockNetIOFail.Error(), diff --git a/network/transparent_endpoint_linux_test.go b/network/transparent_endpoint_linux_test.go index e140dbebb5..09587fa0d5 100644 --- a/network/transparent_endpoint_linux_test.go +++ b/network/transparent_endpoint_linux_test.go @@ -370,7 +370,6 @@ func TestTransConfigureContainerInterfacesAndRoutes(t *testing.T) { Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), }, }, - IsDefaultInterface: true, }, wantErr: true, wantErrMsg: netio.ErrMockNetIOFail.Error(), diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index f72365185b..30385eb855 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -230,7 +230,7 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e } } - if epInfo.IsDefaultInterface { + if !epInfo.SkipDefaultRoutes { // add route for virtualgwip // ip route add 169.254.1.1/32 dev eth0 virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) From cd09e90ce8737b031f8394e7a2d2cb4c38beb78e Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 25 Sep 2023 23:13:57 +0000 Subject: [PATCH 33/64] update tests --- network/secondary_endpoint_client_linux.go | 4 ++++ network/secondary_endpoint_linux_test.go | 27 +++++++++++++--------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index 4d0cf66723..376dadbd19 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -96,6 +96,10 @@ func (client *SecondaryEndpointClient) ConfigureContainerInterfacesAndRoutes(epI return newErrorSecondaryEndpointClient(errors.New(epInfo.IfName + " does not exist")) } + if len(epInfo.Routes) < 1 { + return newErrorSecondaryEndpointClient(errors.New("routes expected for " + epInfo.IfName)) + } + if err := addRoutes(client.netlink, client.netioshim, epInfo.IfName, epInfo.Routes); err != nil { return newErrorSecondaryEndpointClient(err) } diff --git a/network/secondary_endpoint_linux_test.go b/network/secondary_endpoint_linux_test.go index 17ff2dbce8..cc35b46e04 100644 --- a/network/secondary_endpoint_linux_test.go +++ b/network/secondary_endpoint_linux_test.go @@ -150,15 +150,20 @@ func TestSecondaryConfigureContainerInterfacesAndRoutes(t *testing.T) { Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), }, }, + Routes: []RouteInfo{ + { + Dst: net.IPNet{IP: net.ParseIP("192.168.0.4"), Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)}, + }, + }, }, wantErr: false, }, { name: "Configure Interface and routes assign ip fail", client: &SecondaryEndpointClient{ - netlink: netlink.NewMockNetlink(true, "netlink fail"), + netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), - netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netUtilsClient: networkutils.NewNetworkUtils(netlink.NewMockNetlink(true, ""), plc), netioshim: netio.NewMockNetIO(false, 0), ep: &endpoint{SecondaryInterfaces: map[string]*InterfaceInfo{"eth1": {Name: "eth1"}}}, }, @@ -172,7 +177,7 @@ func TestSecondaryConfigureContainerInterfacesAndRoutes(t *testing.T) { }, }, wantErr: true, - wantErrMsg: "netlink fail", + wantErrMsg: "", }, { name: "Configure Interface and routes add routes fail", @@ -191,6 +196,11 @@ func TestSecondaryConfigureContainerInterfacesAndRoutes(t *testing.T) { Mask: net.CIDRMask(subnetv4Mask, ipv4Bits), }, }, + Routes: []RouteInfo{ + { + Dst: net.IPNet{IP: net.ParseIP("192.168.0.4"), Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)}, + }, + }, }, wantErr: true, wantErrMsg: netio.ErrMockNetIOFail.Error(), @@ -217,24 +227,19 @@ func TestSecondaryConfigureContainerInterfacesAndRoutes(t *testing.T) { wantErrMsg: "SecondaryEndpointClient Error: eth2 does not exist", }, { - name: "Configure Interface and routes add routes fail on non default interface", + name: "Configure Interface and routes add routes fail when no routes are provided", client: &SecondaryEndpointClient{ netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 2), + netioshim: netio.NewMockNetIO(false, 0), ep: &endpoint{SecondaryInterfaces: map[string]*InterfaceInfo{"eth1": {Name: "eth1"}}}, }, epInfo: &EndpointInfo{ IfName: "eth1", - Routes: []RouteInfo{ - { - Dst: net.IPNet{IP: net.ParseIP("192.168.0.4"), Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)}, - }, - }, }, wantErr: true, - wantErrMsg: netio.ErrMockNetIOFail.Error(), + wantErrMsg: "SecondaryEndpointClient Error: routes expected for eth1", }, } From 859a8f2c6218647c8fcbf9fe00eb274d0905c0b2 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 27 Sep 2023 22:14:20 +0000 Subject: [PATCH 34/64] only skip adding default route --- network/transparent_endpointclient_linux.go | 68 ++++++++++----------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 30385eb855..1159c1f19a 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -230,18 +230,18 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e } } - if !epInfo.SkipDefaultRoutes { - // add route for virtualgwip - // ip route add 169.254.1.1/32 dev eth0 - virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) - routeInfo := RouteInfo{ - Dst: *virtualGwNet, - Scope: netlink.RT_SCOPE_LINK, - } - if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { - return newErrorTransparentEndpointClient(err) - } + // add route for virtualgwip + // ip route add 169.254.1.1/32 dev eth0 + virtualGwIP, virtualGwNet, _ := net.ParseCIDR(virtualGwIPString) + routeInfo := RouteInfo{ + Dst: *virtualGwNet, + Scope: netlink.RT_SCOPE_LINK, + } + if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { + return newErrorTransparentEndpointClient(err) + } + if !epInfo.SkipDefaultRoutes { // ip route add default via 169.254.1.1 dev eth0 _, defaultIPNet, _ := net.ParseCIDR(defaultGwCidr) dstIP := net.IPNet{IP: net.ParseIP(defaultGw), Mask: defaultIPNet.Mask} @@ -252,37 +252,33 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, []RouteInfo{routeInfo}); err != nil { return err } + } else if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, epInfo.Routes); err != nil { + return newErrorTransparentEndpointClient(err) + } - // arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac - logger.Info("Adding static arp for IP address and MAC in Container namespace", - zap.String("address", virtualGwNet.String()), zap.Any("hostVethMac", client.hostVethMac)) - linkInfo := netlink.LinkInfo{ - Name: client.containerVethName, - IPAddr: virtualGwNet.IP, - MacAddress: client.hostVethMac, - } - - if err := client.netlink.SetOrRemoveLinkAddress(linkInfo, netlink.ADD, netlink.NUD_PROBE); err != nil { - return fmt.Errorf("Adding arp in container failed: %w", err) - } + // arp -s 169.254.1.1 e3:45:f4:ac:34:12 - add static arp entry for virtualgwip to hostveth interface mac + logger.Info("Adding static arp for IP address and MAC in Container namespace", + zap.String("address", virtualGwNet.String()), zap.Any("hostVethMac", client.hostVethMac)) + linkInfo := netlink.LinkInfo{ + Name: client.containerVethName, + IPAddr: virtualGwNet.IP, + MacAddress: client.hostVethMac, + } - // IPv6Mode can be ipv6NAT or dual stack overlay - // set epInfo ipv6Mode to 'dualStackOverlay' to set ipv6Routes and ipv6NeighborEntries for Linux pod in dualStackOverlay ipam mode - if epInfo.IPV6Mode != "" { - if err := client.setupIPV6Routes(); err != nil { - return err - } - } + if err := client.netlink.SetOrRemoveLinkAddress(linkInfo, netlink.ADD, netlink.NUD_PROBE); err != nil { + return fmt.Errorf("Adding arp in container failed: %w", err) + } - if epInfo.IPV6Mode != "" { - return client.setIPV6NeighEntry() + // IPv6Mode can be ipv6NAT or dual stack overlay + // set epInfo ipv6Mode to 'dualStackOverlay' to set ipv6Routes and ipv6NeighborEntries for Linux pod in dualStackOverlay ipam mode + if epInfo.IPV6Mode != "" { + if err := client.setupIPV6Routes(); err != nil { + return err } - - return nil } - if err := addRoutes(client.netlink, client.netioshim, client.containerVethName, epInfo.Routes); err != nil { - return newErrorTransparentEndpointClient(err) + if epInfo.IPV6Mode != "" { + return client.setIPV6NeighEntry() } return nil From 618a260f14f6b568c28bcae673f402f0c7cf8a5b Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 5 Oct 2023 16:25:33 +0000 Subject: [PATCH 35/64] modify AddEndpoints per comments --- network/bridge_endpointclient_linux.go | 3 ++- network/mock_endpointclient.go | 2 +- network/ovs_endpointclient_linux.go | 3 ++- network/secondary_endpoint_client_linux.go | 2 +- network/secondary_endpoint_linux_test.go | 2 +- network/transparent_endpoint_linux_test.go | 2 +- network/transparent_endpointclient_linux.go | 3 ++- network/transparent_vlan_endpointclient_linux.go | 7 ++++--- network/transparent_vlan_endpointclient_linux_test.go | 2 +- 9 files changed, 15 insertions(+), 11 deletions(-) diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go index 15851a4210..649ed17dc2 100644 --- a/network/bridge_endpointclient_linux.go +++ b/network/bridge_endpointclient_linux.go @@ -59,7 +59,7 @@ func NewLinuxBridgeEndpointClient( return client } -func (client *LinuxBridgeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { +func (client *LinuxBridgeEndpointClient) AddEndpoints(epInfo *EndpointInfo, ep *endpoint) error { if err := client.nuc.CreateEndpoint(client.hostVethName, client.containerVethName, nil); err != nil { return err } @@ -70,6 +70,7 @@ func (client *LinuxBridgeEndpointClient) AddEndpoints(epInfo *EndpointInfo) erro } client.containerMac = containerIf.HardwareAddr + ep.MacAddress = containerIf.HardwareAddr return nil } diff --git a/network/mock_endpointclient.go b/network/mock_endpointclient.go index deff106f3c..506ac8fa17 100644 --- a/network/mock_endpointclient.go +++ b/network/mock_endpointclient.go @@ -25,7 +25,7 @@ func NewMockEndpointClient(returnError bool) *MockEndpointClient { return client } -func (client *MockEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { +func (client *MockEndpointClient) AddEndpoints(epInfo *EndpointInfo, _ *endpoint) error { if ok := client.endpoints[epInfo.Id]; ok { return newErrorMockEndpointClient("Endpoint already exists") } diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 428e533ab6..ff699d92ce 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -77,7 +77,7 @@ func NewOVSEndpointClient( return client } -func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { +func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo, ep *endpoint) error { epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) if err := epc.CreateEndpoint(client.hostVethName, client.containerVethName, nil); err != nil { return err @@ -90,6 +90,7 @@ func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { } client.containerMac = containerIf.HardwareAddr.String() + ep.MacAddress = containerIf.HardwareAddr if err := client.AddSnatEndpoint(); err != nil { return err diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index 376dadbd19..49cd7be12f 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -39,7 +39,7 @@ func NewSecondaryEndpointClient( return client } -func (client *SecondaryEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { +func (client *SecondaryEndpointClient) AddEndpoints(epInfo *EndpointInfo, _ *endpoint) error { iface, err := client.netioshim.GetNetworkInterfaceByMac(epInfo.MacAddress) if err != nil { return newErrorSecondaryEndpointClient(err) diff --git a/network/secondary_endpoint_linux_test.go b/network/secondary_endpoint_linux_test.go index cc35b46e04..ac6d3c756f 100644 --- a/network/secondary_endpoint_linux_test.go +++ b/network/secondary_endpoint_linux_test.go @@ -70,7 +70,7 @@ func TestSecondaryAddEndpoints(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.AddEndpoints(tt.epInfo) + err := tt.client.AddEndpoints(tt.epInfo, nil) if tt.wantErr { require.Error(t, err) require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) diff --git a/network/transparent_endpoint_linux_test.go b/network/transparent_endpoint_linux_test.go index 09587fa0d5..15bf7ab921 100644 --- a/network/transparent_endpoint_linux_test.go +++ b/network/transparent_endpoint_linux_test.go @@ -123,7 +123,7 @@ func TestTransAddEndpoints(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.AddEndpoints(tt.epInfo) + err := tt.client.AddEndpoints(tt.epInfo, &endpoint{}) if tt.wantErr { require.Error(t, err) require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 1159c1f19a..ec0d4d33b9 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -76,7 +76,7 @@ func (client *TransparentEndpointClient) setArpProxy(ifName string) error { return err } -func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { +func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo, ep *endpoint) error { if _, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName); err == nil { logger.Info("Deleting old host veth", zap.String("hostVethName", client.hostVethName)) if err = client.netlink.DeleteLink(client.hostVethName); err != nil { @@ -113,6 +113,7 @@ func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) erro } client.containerMac = containerIf.HardwareAddr + ep.MacAddress = containerIf.HardwareAddr hostVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName) if err != nil { diff --git a/network/transparent_vlan_endpointclient_linux.go b/network/transparent_vlan_endpointclient_linux.go index 19363bbfb0..2f9cef77a3 100644 --- a/network/transparent_vlan_endpointclient_linux.go +++ b/network/transparent_vlan_endpointclient_linux.go @@ -108,9 +108,9 @@ func NewTransparentVlanEndpointClient( } // Adds interfaces to the vnet (created if not existing) and vm namespace -func (client *TransparentVlanEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { +func (client *TransparentVlanEndpointClient) AddEndpoints(epInfo *EndpointInfo, ep *endpoint) error { // VM Namespace - err := client.PopulateVM(epInfo) + err := client.PopulateVM(epInfo, ep) if err != nil { return err } @@ -153,7 +153,7 @@ func (client *TransparentVlanEndpointClient) createNetworkNamespace(vmNS, numRet } // Called from AddEndpoints, Namespace: VM -func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) error { +func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo, ep *endpoint) error { vmNS, err := client.netnsClient.Get() if err != nil { return errors.Wrap(err, "failed to get vm ns handle") @@ -283,6 +283,7 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er return errors.Wrap(err, "container veth does not exist") } client.containerMac = containerIf.HardwareAddr + ep.MacAddress = containerIf.HardwareAddr return nil } diff --git a/network/transparent_vlan_endpointclient_linux_test.go b/network/transparent_vlan_endpointclient_linux_test.go index 9600236324..1411be6a67 100644 --- a/network/transparent_vlan_endpointclient_linux_test.go +++ b/network/transparent_vlan_endpointclient_linux_test.go @@ -267,7 +267,7 @@ func TestTransparentVlanAddEndpoints(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.PopulateVM(tt.epInfo) + err := tt.client.PopulateVM(tt.epInfo, &endpoint{}) if tt.wantErr { require.Error(t, err) require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) From 67d35b99c83ac5cc596031d475ea2c34b41a341b Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 9 Oct 2023 22:05:48 +0000 Subject: [PATCH 36/64] address comments --- network/bridge_endpointclient_linux.go | 3 +- network/mock_endpointclient.go | 34 +++++++++++-------- network/ovs_endpointclient_linux.go | 3 +- network/secondary_endpoint_client_linux.go | 18 +++++++--- network/secondary_endpoint_linux_test.go | 2 +- network/transparent_endpoint_linux_test.go | 2 +- network/transparent_endpointclient_linux.go | 3 +- .../transparent_vlan_endpointclient_linux.go | 7 ++-- ...nsparent_vlan_endpointclient_linux_test.go | 2 +- 9 files changed, 43 insertions(+), 31 deletions(-) diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go index 649ed17dc2..15851a4210 100644 --- a/network/bridge_endpointclient_linux.go +++ b/network/bridge_endpointclient_linux.go @@ -59,7 +59,7 @@ func NewLinuxBridgeEndpointClient( return client } -func (client *LinuxBridgeEndpointClient) AddEndpoints(epInfo *EndpointInfo, ep *endpoint) error { +func (client *LinuxBridgeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if err := client.nuc.CreateEndpoint(client.hostVethName, client.containerVethName, nil); err != nil { return err } @@ -70,7 +70,6 @@ func (client *LinuxBridgeEndpointClient) AddEndpoints(epInfo *EndpointInfo, ep * } client.containerMac = containerIf.HardwareAddr - ep.MacAddress = containerIf.HardwareAddr return nil } diff --git a/network/mock_endpointclient.go b/network/mock_endpointclient.go index 506ac8fa17..b95ab47890 100644 --- a/network/mock_endpointclient.go +++ b/network/mock_endpointclient.go @@ -7,36 +7,42 @@ import ( var errMockEpClient = errors.New("MockEndpointClient Error") -func newErrorMockEndpointClient(errStr string) error { +const ( + eth0IfName = "eth0" +) + +func NewErrorMockEndpointClient(errStr string) error { return fmt.Errorf("%w : %s", errMockEpClient, errStr) } type MockEndpointClient struct { - endpoints map[string]bool - returnError bool + endpoints map[string]bool + testAddEndpointFn func(*EndpointInfo) error } -func NewMockEndpointClient(returnError bool) *MockEndpointClient { +func NewMockEndpointClient(fn func(*EndpointInfo) error) *MockEndpointClient { + if fn == nil { + fn = func(_ *EndpointInfo) error { + return nil + } + } + client := &MockEndpointClient{ - endpoints: make(map[string]bool), - returnError: returnError, + endpoints: make(map[string]bool), + testAddEndpointFn: fn, } return client } -func (client *MockEndpointClient) AddEndpoints(epInfo *EndpointInfo, _ *endpoint) error { - if ok := client.endpoints[epInfo.Id]; ok { - return newErrorMockEndpointClient("Endpoint already exists") +func (client *MockEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { + if ok := client.endpoints[epInfo.Id]; ok && epInfo.IfName == eth0IfName { + return NewErrorMockEndpointClient("Endpoint already exists") } client.endpoints[epInfo.Id] = true - if client.returnError { - return newErrorMockEndpointClient("AddEndpoints failed") - } - - return nil + return client.testAddEndpointFn(epInfo) } func (client *MockEndpointClient) AddEndpointRules(_ *EndpointInfo) error { diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index ff699d92ce..428e533ab6 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -77,7 +77,7 @@ func NewOVSEndpointClient( return client } -func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo, ep *endpoint) error { +func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { epc := networkutils.NewNetworkUtils(client.netlink, client.plClient) if err := epc.CreateEndpoint(client.hostVethName, client.containerVethName, nil); err != nil { return err @@ -90,7 +90,6 @@ func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo, ep *endpoint } client.containerMac = containerIf.HardwareAddr.String() - ep.MacAddress = containerIf.HardwareAddr if err := client.AddSnatEndpoint(); err != nil { return err diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index 49cd7be12f..fc866c6821 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -4,6 +4,7 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" "github.com/pkg/errors" @@ -39,7 +40,7 @@ func NewSecondaryEndpointClient( return client } -func (client *SecondaryEndpointClient) AddEndpoints(epInfo *EndpointInfo, _ *endpoint) error { +func (client *SecondaryEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { iface, err := client.netioshim.GetNetworkInterfaceByMac(epInfo.MacAddress) if err != nil { return newErrorSecondaryEndpointClient(err) @@ -109,8 +110,17 @@ func (client *SecondaryEndpointClient) ConfigureContainerInterfacesAndRoutes(epI return nil } -func (client *SecondaryEndpointClient) DeleteEndpoints(_ *endpoint) error { - // TO-DO: try to clean up and move back to default ns? - // looks like interface goes back to default state (down without routes) after deleting pod +func (client *SecondaryEndpointClient) DeleteEndpoints(ep *endpoint) error { + netnsClient := netns.New() + vmns, err := netnsClient.Get() + if err != nil { + return newErrorSecondaryEndpointClient(err) + } + + for iface := range ep.SecondaryInterfaces { + if err := client.netlink.SetLinkNetNs(iface, uintptr(vmns)); err != nil { + return newErrorSecondaryEndpointClient(err) + } + } return nil } diff --git a/network/secondary_endpoint_linux_test.go b/network/secondary_endpoint_linux_test.go index ac6d3c756f..cc35b46e04 100644 --- a/network/secondary_endpoint_linux_test.go +++ b/network/secondary_endpoint_linux_test.go @@ -70,7 +70,7 @@ func TestSecondaryAddEndpoints(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.AddEndpoints(tt.epInfo, nil) + err := tt.client.AddEndpoints(tt.epInfo) if tt.wantErr { require.Error(t, err) require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) diff --git a/network/transparent_endpoint_linux_test.go b/network/transparent_endpoint_linux_test.go index 15bf7ab921..09587fa0d5 100644 --- a/network/transparent_endpoint_linux_test.go +++ b/network/transparent_endpoint_linux_test.go @@ -123,7 +123,7 @@ func TestTransAddEndpoints(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.AddEndpoints(tt.epInfo, &endpoint{}) + err := tt.client.AddEndpoints(tt.epInfo) if tt.wantErr { require.Error(t, err) require.Contains(t, tt.wantErrMsg, err.Error(), "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index ec0d4d33b9..1159c1f19a 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -76,7 +76,7 @@ func (client *TransparentEndpointClient) setArpProxy(ifName string) error { return err } -func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo, ep *endpoint) error { +func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if _, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName); err == nil { logger.Info("Deleting old host veth", zap.String("hostVethName", client.hostVethName)) if err = client.netlink.DeleteLink(client.hostVethName); err != nil { @@ -113,7 +113,6 @@ func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo, ep * } client.containerMac = containerIf.HardwareAddr - ep.MacAddress = containerIf.HardwareAddr hostVethIf, err := client.netioshim.GetNetworkInterfaceByName(client.hostVethName) if err != nil { diff --git a/network/transparent_vlan_endpointclient_linux.go b/network/transparent_vlan_endpointclient_linux.go index 2f9cef77a3..19363bbfb0 100644 --- a/network/transparent_vlan_endpointclient_linux.go +++ b/network/transparent_vlan_endpointclient_linux.go @@ -108,9 +108,9 @@ func NewTransparentVlanEndpointClient( } // Adds interfaces to the vnet (created if not existing) and vm namespace -func (client *TransparentVlanEndpointClient) AddEndpoints(epInfo *EndpointInfo, ep *endpoint) error { +func (client *TransparentVlanEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { // VM Namespace - err := client.PopulateVM(epInfo, ep) + err := client.PopulateVM(epInfo) if err != nil { return err } @@ -153,7 +153,7 @@ func (client *TransparentVlanEndpointClient) createNetworkNamespace(vmNS, numRet } // Called from AddEndpoints, Namespace: VM -func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo, ep *endpoint) error { +func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) error { vmNS, err := client.netnsClient.Get() if err != nil { return errors.Wrap(err, "failed to get vm ns handle") @@ -283,7 +283,6 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo, ep return errors.Wrap(err, "container veth does not exist") } client.containerMac = containerIf.HardwareAddr - ep.MacAddress = containerIf.HardwareAddr return nil } diff --git a/network/transparent_vlan_endpointclient_linux_test.go b/network/transparent_vlan_endpointclient_linux_test.go index 1411be6a67..9600236324 100644 --- a/network/transparent_vlan_endpointclient_linux_test.go +++ b/network/transparent_vlan_endpointclient_linux_test.go @@ -267,7 +267,7 @@ func TestTransparentVlanAddEndpoints(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - err := tt.client.PopulateVM(tt.epInfo, &endpoint{}) + err := tt.client.PopulateVM(tt.epInfo) if tt.wantErr { require.Error(t, err) require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) From ead5b7b9010ef9bda29cf25b82c1937c246599e2 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Tue, 10 Oct 2023 15:27:41 +0000 Subject: [PATCH 37/64] update deleteendpoint --- network/secondary_endpoint_client_linux.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index fc866c6821..dcc8641a46 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -111,16 +111,18 @@ func (client *SecondaryEndpointClient) ConfigureContainerInterfacesAndRoutes(epI } func (client *SecondaryEndpointClient) DeleteEndpoints(ep *endpoint) error { - netnsClient := netns.New() - vmns, err := netnsClient.Get() + vmns, err := netns.New().Get() if err != nil { return newErrorSecondaryEndpointClient(err) } for iface := range ep.SecondaryInterfaces { if err := client.netlink.SetLinkNetNs(iface, uintptr(vmns)); err != nil { - return newErrorSecondaryEndpointClient(err) + log.Errorf("%w", newErrorSecondaryEndpointClient(err)) } + + delete(ep.SecondaryInterfaces, iface) } + return nil } From ea11100061aa20bc3a3957feb0979e684a02747c Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 11 Oct 2023 15:34:50 +0000 Subject: [PATCH 38/64] enter ns before moving interface back to vm ns --- network/secondary_endpoint_client_linux.go | 47 +++++++++++++++++----- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index dcc8641a46..94a4861cdc 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -1,13 +1,13 @@ package network import ( - "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/netns" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" "github.com/pkg/errors" + "go.uber.org/zap" ) var errorSecondaryEndpointClient = errors.New("SecondaryEndpointClient Error") @@ -70,7 +70,7 @@ func (client *SecondaryEndpointClient) DeleteEndpointRules(_ *endpoint) { func (client *SecondaryEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { // Move the container interface to container's network namespace. - log.Printf("[net] Setting link %v netns %v.", epInfo.IfName, epInfo.NetNsPath) + logger.Info("[net] Setting link %v netns %v.", zap.String("IfName", epInfo.IfName), zap.String("NetNsPath", epInfo.NetNsPath)) if err := client.netlink.SetLinkNetNs(epInfo.IfName, nsID); err != nil { return newErrorSecondaryEndpointClient(err) } @@ -79,7 +79,7 @@ func (client *SecondaryEndpointClient) MoveEndpointsToContainerNS(epInfo *Endpoi } func (client *SecondaryEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - log.Printf("[net] Setting link %v state up.", epInfo.IfName) + logger.Info("[net] Setting link state up.", zap.String("IfName", epInfo.IfName)) if err := client.netlink.SetLinkState(epInfo.IfName, true); err != nil { return newErrorSecondaryEndpointClient(err) } @@ -111,17 +111,42 @@ func (client *SecondaryEndpointClient) ConfigureContainerInterfacesAndRoutes(epI } func (client *SecondaryEndpointClient) DeleteEndpoints(ep *endpoint) error { - vmns, err := netns.New().Get() - if err != nil { - return newErrorSecondaryEndpointClient(err) - } + if ep.NetworkNameSpace != "" { + // Get VM namespace + vmns, err := netns.New().Get() + if err != nil { + return newErrorSecondaryEndpointClient(err) + } - for iface := range ep.SecondaryInterfaces { - if err := client.netlink.SetLinkNetNs(iface, uintptr(vmns)); err != nil { - log.Errorf("%w", newErrorSecondaryEndpointClient(err)) + // Open the network namespace. + logger.Info("Opening netns", zap.Any("NetNsPath", ep.NetworkNameSpace)) + ns, err := OpenNamespace(ep.NetworkNameSpace) + if err != nil { + return err } + defer ns.Close() - delete(ep.SecondaryInterfaces, iface) + // Enter the container network namespace. + logger.Info("Entering netns", zap.Any("NetNsPath", ep.NetworkNameSpace)) + if err := ns.Enter(); err != nil { + return err + } + + // Return to host network namespace. + defer func() { + logger.Info("Exiting netns", zap.Any("NetNsPath", ep.NetworkNameSpace)) + if err := ns.Exit(); err != nil { + logger.Error("Failed to exit netns with", zap.Error(newErrorSecondaryEndpointClient(err))) + } + }() + + for iface := range ep.SecondaryInterfaces { + if err := client.netlink.SetLinkNetNs(iface, uintptr(vmns)); err != nil { + logger.Error("Failed to move interface", zap.String("IfName", iface), zap.Error(newErrorSecondaryEndpointClient(err))) + } + + delete(ep.SecondaryInterfaces, iface) + } } return nil From 98ed253bc4e5a134ea28f23d414527daa7a3a168 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 13 Oct 2023 20:13:20 +0000 Subject: [PATCH 39/64] update delete endpoint test --- network/secondary_endpoint_linux_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/network/secondary_endpoint_linux_test.go b/network/secondary_endpoint_linux_test.go index cc35b46e04..55a62a7c30 100644 --- a/network/secondary_endpoint_linux_test.go +++ b/network/secondary_endpoint_linux_test.go @@ -4,7 +4,9 @@ package network import ( + "fmt" "net" + "os" "testing" "github.com/Azure/azure-container-networking/netio" @@ -12,6 +14,7 @@ import ( "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" ) func TestSecondaryAddEndpoints(t *testing.T) { @@ -82,7 +85,7 @@ func TestSecondaryAddEndpoints(t *testing.T) { } } -func TestSecondaryDeleteEndpointsRules(t *testing.T) { +func TestSecondaryDeleteEndpoints(t *testing.T) { nl := netlink.NewMockNetlink(false, "") plc := platform.NewMockExecClient(false) @@ -92,7 +95,7 @@ func TestSecondaryDeleteEndpointsRules(t *testing.T) { ep *endpoint }{ { - name: "Delete endpoint rules happy path", + name: "Delete endpoint happy path", client: &SecondaryEndpointClient{ netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), @@ -100,6 +103,7 @@ func TestSecondaryDeleteEndpointsRules(t *testing.T) { netioshim: netio.NewMockNetIO(false, 0), }, ep: &endpoint{ + NetworkNameSpace: fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid()), SecondaryInterfaces: map[string]*InterfaceInfo{ "eth1": { Name: "eth1", @@ -117,7 +121,9 @@ func TestSecondaryDeleteEndpointsRules(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - tt.client.DeleteEndpointRules(tt.ep) + require.Len(t, tt.ep.SecondaryInterfaces, 1) + require.Nil(t, tt.client.DeleteEndpoints(tt.ep)) + require.Len(t, tt.ep.SecondaryInterfaces, 0) }) } } From 9b76bfbd884ed1bfdbefb29213fa5c4144fefa9f Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 18 Oct 2023 15:12:34 +0000 Subject: [PATCH 40/64] add namespace interface for testing --- network/mock_namespace.go | 51 ++++++++++++++ network/namespace.go | 14 ++++ network/namespace_linux.go | 29 +++++--- network/namespace_windows.go | 61 +++++++++++++++++ network/secondary_endpoint_client_linux.go | 68 +++++++++++-------- network/secondary_endpoint_linux_test.go | 28 ++++++-- .../transparent_vlan_endpointclient_linux.go | 27 ++++---- 7 files changed, 224 insertions(+), 54 deletions(-) create mode 100644 network/mock_namespace.go create mode 100644 network/namespace.go create mode 100644 network/namespace_windows.go diff --git a/network/mock_namespace.go b/network/mock_namespace.go new file mode 100644 index 0000000000..8a82109bdb --- /dev/null +++ b/network/mock_namespace.go @@ -0,0 +1,51 @@ +// Copyright 2017 Microsoft. All rights reserved. +// MIT License + +package network + +import "errors" + +type MockNamespace struct{} + +type MockNamespaceClient struct{} + +func NewMockNamespaceClient() *MockNamespaceClient { + return &MockNamespaceClient{} +} + +// OpenNamespace creates a new namespace object for the given netns path. +func (c *MockNamespaceClient) OpenNamespace(ns string) (NamespaceInterface, error) { + if ns == "" { + return nil, errors.New("no such file or directory") + } + return &MockNamespace{}, nil +} + +// GetCurrentThreadNamespace returns the caller thread's current namespace. +func (c *MockNamespaceClient) GetCurrentThreadNamespace() (NamespaceInterface, error) { + return c.OpenNamespace("") +} + +// Close releases the resources associated with the namespace object. +func (ns *MockNamespace) Close() error { + return nil +} + +// GetFd returns the file descriptor of the namespace. +func (ns *MockNamespace) GetFd() uintptr { + return 1 +} + +func (ns *MockNamespace) GetName() string { + return "nsname" +} + +// Enter puts the caller thread inside the namespace. +func (ns *MockNamespace) Enter() error { + return nil +} + +// Exit puts the caller thread to its previous namespace. +func (ns *MockNamespace) Exit() error { + return nil +} diff --git a/network/namespace.go b/network/namespace.go new file mode 100644 index 0000000000..ca505145b0 --- /dev/null +++ b/network/namespace.go @@ -0,0 +1,14 @@ +package network + +type NamespaceInterface interface { + GetFd() uintptr + GetName() string + Enter() error + Exit() error + Close() error +} + +type NamespaceClientInterface interface { + OpenNamespace(nsPath string) (NamespaceInterface, error) + GetCurrentThreadNamespace() (NamespaceInterface, error) +} diff --git a/network/namespace_linux.go b/network/namespace_linux.go index 455bf4b5c7..bc682ded3b 100644 --- a/network/namespace_linux.go +++ b/network/namespace_linux.go @@ -17,22 +17,29 @@ import ( type Namespace struct { file *os.File prevNs *Namespace + cli *NamespaceClient +} + +type NamespaceClient struct{} + +func NewNamespaceClient() *NamespaceClient { + return &NamespaceClient{} } // OpenNamespace creates a new namespace object for the given netns path. -func OpenNamespace(nsPath string) (*Namespace, error) { +func (c *NamespaceClient) OpenNamespace(nsPath string) (NamespaceInterface, error) { fd, err := os.Open(nsPath) if err != nil { return nil, err } - return &Namespace{file: fd}, nil + return &Namespace{file: fd, cli: c}, nil } // GetCurrentThreadNamespace returns the caller thread's current namespace. -func GetCurrentThreadNamespace() (*Namespace, error) { +func (c *NamespaceClient) GetCurrentThreadNamespace() (NamespaceInterface, error) { nsPath := fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid()) - return OpenNamespace(nsPath) + return c.OpenNamespace(nsPath) } // Close releases the resources associated with the namespace object. @@ -43,7 +50,7 @@ func (ns *Namespace) Close() error { err := ns.file.Close() if err != nil { - return fmt.Errorf("Failed to close namespace %v, err:%v", ns.file.Name(), err) + return fmt.Errorf("failed to close namespace %v, err:%w", ns.file.Name(), err) } ns.file = nil @@ -56,11 +63,15 @@ func (ns *Namespace) GetFd() uintptr { return ns.file.Fd() } +func (ns *Namespace) GetName() string { + return ns.file.Name() +} + // Set sets the current namespace. func (ns *Namespace) set() error { _, _, err := unix.Syscall(unix.SYS_SETNS, ns.file.Fd(), uintptr(unix.CLONE_NEWNET), 0) if err != 0 { - return fmt.Errorf("Failed to set namespace %v, err:%v", ns.file.Name(), err) + return fmt.Errorf("failed to set namespace %v, err:%w", ns.file.Name(), err) } return nil @@ -68,13 +79,13 @@ func (ns *Namespace) set() error { // Enter puts the caller thread inside the namespace. func (ns *Namespace) Enter() error { - var err error - - ns.prevNs, err = GetCurrentThreadNamespace() + currentNs, err := ns.cli.GetCurrentThreadNamespace() if err != nil { return err } + ns.prevNs = currentNs.(*Namespace) + runtime.LockOSThread() err = ns.set() diff --git a/network/namespace_windows.go b/network/namespace_windows.go new file mode 100644 index 0000000000..85390a3cb5 --- /dev/null +++ b/network/namespace_windows.go @@ -0,0 +1,61 @@ +// Copyright 2017 Microsoft. All rights reserved. +// MIT License + +package network + +import ( + "errors" + "os" +) + +// Namespace represents a network namespace. +type Namespace struct { + file *os.File + prevNs *Namespace + cli *NamespaceClient +} + +type NamespaceClient struct{} + +func NewNamespaceClient() *NamespaceClient { + return &NamespaceClient{} +} + +// OpenNamespace creates a new namespace object for the given netns path. +func (c *NamespaceClient) OpenNamespace(nsPath string) (NamespaceInterface, error) { + return nil, errors.New("windows impl") +} + +// GetCurrentThreadNamespace returns the caller thread's current namespace. +func (c *NamespaceClient) GetCurrentThreadNamespace() (NamespaceInterface, error) { + return c.OpenNamespace("") +} + +// Close releases the resources associated with the namespace object. +func (ns *Namespace) Close() error { + return nil +} + +// GetFd returns the file descriptor of the namespace. +func (ns *Namespace) GetFd() uintptr { + return 0 +} + +func (ns *Namespace) GetName() string { + return "windows impl" +} + +// Set sets the current namespace. +func (ns *Namespace) set() error { + return nil +} + +// Enter puts the caller thread inside the namespace. +func (ns *Namespace) Enter() error { + return nil +} + +// Exit puts the caller thread to its previous namespace. +func (ns *Namespace) Exit() error { + return nil +} diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index 94a4861cdc..0667b03bca 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -1,6 +1,8 @@ package network import ( + "strings" + "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/netns" @@ -12,6 +14,8 @@ import ( var errorSecondaryEndpointClient = errors.New("SecondaryEndpointClient Error") +const errFileNotExist = "no such file or directory" + func newErrorSecondaryEndpointClient(err error) error { return errors.Wrapf(err, "%s", errorSecondaryEndpointClient) } @@ -21,12 +25,14 @@ type SecondaryEndpointClient struct { netioshim netio.NetIOInterface plClient platform.ExecClient netUtilsClient networkutils.NetworkUtils + nsClient NamespaceClientInterface ep *endpoint } func NewSecondaryEndpointClient( nl netlink.NetlinkInterface, plc platform.ExecClient, + nsc NamespaceClientInterface, endpoint *endpoint, ) *SecondaryEndpointClient { client := &SecondaryEndpointClient{ @@ -34,6 +40,7 @@ func NewSecondaryEndpointClient( netioshim: &netio.NetIO{}, plClient: plc, netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + nsClient: nsc, ep: endpoint, } @@ -111,42 +118,45 @@ func (client *SecondaryEndpointClient) ConfigureContainerInterfacesAndRoutes(epI } func (client *SecondaryEndpointClient) DeleteEndpoints(ep *endpoint) error { - if ep.NetworkNameSpace != "" { - // Get VM namespace - vmns, err := netns.New().Get() - if err != nil { - return newErrorSecondaryEndpointClient(err) - } + // Get VM namespace + vmns, err := netns.New().Get() + if err != nil { + return newErrorSecondaryEndpointClient(err) + } - // Open the network namespace. - logger.Info("Opening netns", zap.Any("NetNsPath", ep.NetworkNameSpace)) - ns, err := OpenNamespace(ep.NetworkNameSpace) - if err != nil { - return err + // Open the network namespace. + logger.Info("Opening netns", zap.Any("NetNsPath", ep.NetworkNameSpace)) + ns, err := client.nsClient.OpenNamespace(ep.NetworkNameSpace) + if err != nil { + if strings.Contains(err.Error(), errFileNotExist) { + ep.SecondaryInterfaces = make(map[string]*InterfaceInfo) + return nil } - defer ns.Close() - // Enter the container network namespace. - logger.Info("Entering netns", zap.Any("NetNsPath", ep.NetworkNameSpace)) - if err := ns.Enter(); err != nil { - return err - } + return newErrorSecondaryEndpointClient(err) + } + defer ns.Close() - // Return to host network namespace. - defer func() { - logger.Info("Exiting netns", zap.Any("NetNsPath", ep.NetworkNameSpace)) - if err := ns.Exit(); err != nil { - logger.Error("Failed to exit netns with", zap.Error(newErrorSecondaryEndpointClient(err))) - } - }() + // Enter the container network namespace. + logger.Info("Entering netns", zap.Any("NetNsPath", ep.NetworkNameSpace)) + if err := ns.Enter(); err != nil { + return newErrorSecondaryEndpointClient(err) + } - for iface := range ep.SecondaryInterfaces { - if err := client.netlink.SetLinkNetNs(iface, uintptr(vmns)); err != nil { - logger.Error("Failed to move interface", zap.String("IfName", iface), zap.Error(newErrorSecondaryEndpointClient(err))) - } + // Return to host network namespace. + defer func() { + logger.Info("Exiting netns", zap.Any("NetNsPath", ep.NetworkNameSpace)) + if err := ns.Exit(); err != nil { + logger.Error("Failed to exit netns with", zap.Error(newErrorSecondaryEndpointClient(err))) + } + }() - delete(ep.SecondaryInterfaces, iface) + for iface := range ep.SecondaryInterfaces { + if err := client.netlink.SetLinkNetNs(iface, uintptr(vmns)); err != nil { + logger.Error("Failed to move interface", zap.String("IfName", iface), zap.Error(newErrorSecondaryEndpointClient(err))) } + + delete(ep.SecondaryInterfaces, iface) } return nil diff --git a/network/secondary_endpoint_linux_test.go b/network/secondary_endpoint_linux_test.go index 55a62a7c30..8a54894e49 100644 --- a/network/secondary_endpoint_linux_test.go +++ b/network/secondary_endpoint_linux_test.go @@ -4,9 +4,7 @@ package network import ( - "fmt" "net" - "os" "testing" "github.com/Azure/azure-container-networking/netio" @@ -14,7 +12,6 @@ import ( "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/platform" "github.com/stretchr/testify/require" - "golang.org/x/sys/unix" ) func TestSecondaryAddEndpoints(t *testing.T) { @@ -101,9 +98,32 @@ func TestSecondaryDeleteEndpoints(t *testing.T) { plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), + }, + ep: &endpoint{ + NetworkNameSpace: "testns", + SecondaryInterfaces: map[string]*InterfaceInfo{ + "eth1": { + Name: "eth1", + Routes: []RouteInfo{ + { + Dst: net.IPNet{IP: net.ParseIP("192.168.0.4"), Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)}, + }, + }, + }, + }, + }, + }, + { + name: "Delete endpoint happy path namespace not found", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), }, ep: &endpoint{ - NetworkNameSpace: fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid()), SecondaryInterfaces: map[string]*InterfaceInfo{ "eth1": { Name: "eth1", diff --git a/network/transparent_vlan_endpointclient_linux.go b/network/transparent_vlan_endpointclient_linux.go index 19363bbfb0..5532db20d1 100644 --- a/network/transparent_vlan_endpointclient_linux.go +++ b/network/transparent_vlan_endpointclient_linux.go @@ -68,6 +68,7 @@ type TransparentVlanEndpointClient struct { netioshim netio.NetIOInterface plClient platform.ExecClient netUtilsClient networkutils.NetworkUtils + nsClient NamespaceClientInterface } func NewTransparentVlanEndpointClient( @@ -79,6 +80,7 @@ func NewTransparentVlanEndpointClient( localIP string, nl netlink.NetlinkInterface, plc platform.ExecClient, + nsc NamespaceClientInterface, ) *TransparentVlanEndpointClient { vlanVethName := fmt.Sprintf("%s_%d", nw.extIf.Name, vlanid) vnetNSName := fmt.Sprintf("az_ns_%d", vlanid) @@ -100,6 +102,7 @@ func NewTransparentVlanEndpointClient( netioshim: &netio.NetIO{}, plClient: plc, netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + nsClient: nsc, } client.NewSnatClient(nw.SnatBridgeIP, localIP, ep) @@ -118,7 +121,7 @@ func (client *TransparentVlanEndpointClient) AddEndpoints(epInfo *EndpointInfo) return errors.Wrap(err, "failed to add snat endpoint") } // VNET Namespace - return ExecuteInNS(client.vnetNSName, func() error { + return ExecuteInNS(client.nsClient, client.vnetNSName, func() error { return client.PopulateVnet(epInfo) }) } @@ -315,7 +318,7 @@ func (client *TransparentVlanEndpointClient) AddEndpointRules(epInfo *EndpointIn return errors.Wrap(err, "failed to add snat endpoint rules") } logger.Info("[transparent-vlan] Adding tunneling rules in vnet namespace") - err := ExecuteInNS(client.vnetNSName, func() error { + err := ExecuteInNS(client.nsClient, client.vnetNSName, func() error { return client.AddVnetRules(epInfo) }) return err @@ -392,7 +395,7 @@ func (client *TransparentVlanEndpointClient) ConfigureContainerInterfacesAndRout } // Switch to vnet NS and call ConfigureVnetInterfacesAndRoutes - err = ExecuteInNS(client.vnetNSName, func() error { + err = ExecuteInNS(client.nsClient, client.vnetNSName, func() error { return client.ConfigureVnetInterfacesAndRoutesImpl(epInfo) }) if err != nil { @@ -544,7 +547,7 @@ func (client *TransparentVlanEndpointClient) AddDefaultArp(interfaceName, destMa func (client *TransparentVlanEndpointClient) DeleteEndpoints(ep *endpoint) error { // Vnet NS - err := ExecuteInNS(client.vnetNSName, func() error { + err := ExecuteInNS(client.nsClient, client.vnetNSName, func() error { // Passing in functionality to get number of routes after deletion getNumRoutesLeft := func() (int, error) { routes, err := vishnetlink.RouteList(nil, vishnetlink.FAMILY_V4) @@ -597,39 +600,39 @@ func (client *TransparentVlanEndpointClient) DeleteEndpointsImpl(ep *endpoint, _ // Helper function that allows executing a function in a VM namespace // Does not work for process namespaces -func ExecuteInNS(nsName string, f func() error) error { +func ExecuteInNS(nsc NamespaceClientInterface, nsName string, f func() error) error { // Current namespace - returnedTo, err := GetCurrentThreadNamespace() + returnedTo, err := nsc.GetCurrentThreadNamespace() if err != nil { logger.Error("[ExecuteInNS] Could not get NS we are in", zap.Error(err)) } else { - logger.Info("[ExecuteInNS] In NS before switch", zap.String("fileName", returnedTo.file.Name())) + logger.Info("[ExecuteInNS] In NS before switch", zap.String("fileName", returnedTo.GetName())) } // Open the network namespace logger.Info("[ExecuteInNS] Opening ns", zap.String("nsName", fmt.Sprintf("/var/run/netns/%s", nsName))) - ns, err := OpenNamespace(fmt.Sprintf("/var/run/netns/%s", nsName)) + ns, err := nsc.OpenNamespace(fmt.Sprintf("/var/run/netns/%s", nsName)) if err != nil { return err } defer ns.Close() // Enter the network namespace - logger.Info("[ExecuteInNS] Entering ns", zap.String("nsFileName", ns.file.Name())) + logger.Info("[ExecuteInNS] Entering ns", zap.String("nsFileName", ns.GetName())) if err := ns.Enter(); err != nil { return err } // Exit network namespace defer func() { - logger.Info("[ExecuteInNS] Exiting ns", zap.String("nsFileName", ns.file.Name())) + logger.Info("[ExecuteInNS] Exiting ns", zap.String("nsFileName", ns.GetName())) if err := ns.Exit(); err != nil { logger.Error("[ExecuteInNS] Could not exit ns", zap.Error(err)) } - returnedTo, err := GetCurrentThreadNamespace() + returnedTo, err := nsc.GetCurrentThreadNamespace() if err != nil { logger.Error("[ExecuteInNS] Could not get NS we returned to", zap.Error(err)) } else { - logger.Info("[ExecuteInNS] Returned to NS", zap.String("fileName", returnedTo.file.Name())) + logger.Info("[ExecuteInNS] Returned to NS", zap.String("fileName", returnedTo.GetName())) } }() return f() From d5fc773937157a6729bc839233d96cedb183068a Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 18 Oct 2023 18:20:39 +0000 Subject: [PATCH 41/64] fix lint --- network/mock_namespace.go | 4 +--- network/namespace_windows.go | 18 +++++------------- network/secondary_endpoint_client_linux.go | 9 +++++---- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/network/mock_namespace.go b/network/mock_namespace.go index 8a82109bdb..1d7b558624 100644 --- a/network/mock_namespace.go +++ b/network/mock_namespace.go @@ -3,8 +3,6 @@ package network -import "errors" - type MockNamespace struct{} type MockNamespaceClient struct{} @@ -16,7 +14,7 @@ func NewMockNamespaceClient() *MockNamespaceClient { // OpenNamespace creates a new namespace object for the given netns path. func (c *MockNamespaceClient) OpenNamespace(ns string) (NamespaceInterface, error) { if ns == "" { - return nil, errors.New("no such file or directory") + return nil, errFileNotExist } return &MockNamespace{}, nil } diff --git a/network/namespace_windows.go b/network/namespace_windows.go index 85390a3cb5..c60641528d 100644 --- a/network/namespace_windows.go +++ b/network/namespace_windows.go @@ -5,15 +5,12 @@ package network import ( "errors" - "os" ) +const windowsImplErr = errors.New("windows impl err") + // Namespace represents a network namespace. -type Namespace struct { - file *os.File - prevNs *Namespace - cli *NamespaceClient -} +type Namespace struct{} type NamespaceClient struct{} @@ -22,8 +19,8 @@ func NewNamespaceClient() *NamespaceClient { } // OpenNamespace creates a new namespace object for the given netns path. -func (c *NamespaceClient) OpenNamespace(nsPath string) (NamespaceInterface, error) { - return nil, errors.New("windows impl") +func (c *NamespaceClient) OpenNamespace(_ string) (NamespaceInterface, error) { + return nil, windowsImplErr } // GetCurrentThreadNamespace returns the caller thread's current namespace. @@ -45,11 +42,6 @@ func (ns *Namespace) GetName() string { return "windows impl" } -// Set sets the current namespace. -func (ns *Namespace) set() error { - return nil -} - // Enter puts the caller thread inside the namespace. func (ns *Namespace) Enter() error { return nil diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index 0667b03bca..d9c5e7fd67 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -12,9 +12,10 @@ import ( "go.uber.org/zap" ) -var errorSecondaryEndpointClient = errors.New("SecondaryEndpointClient Error") - -const errFileNotExist = "no such file or directory" +var ( + errorSecondaryEndpointClient = errors.New("SecondaryEndpointClient Error") + errFileNotExist = errors.New("no such file or directory") +) func newErrorSecondaryEndpointClient(err error) error { return errors.Wrapf(err, "%s", errorSecondaryEndpointClient) @@ -128,7 +129,7 @@ func (client *SecondaryEndpointClient) DeleteEndpoints(ep *endpoint) error { logger.Info("Opening netns", zap.Any("NetNsPath", ep.NetworkNameSpace)) ns, err := client.nsClient.OpenNamespace(ep.NetworkNameSpace) if err != nil { - if strings.Contains(err.Error(), errFileNotExist) { + if strings.Contains(err.Error(), errFileNotExist.Error()) { ep.SecondaryInterfaces = make(map[string]*InterfaceInfo) return nil } From 62ecb8ab127b19e2aeb534a5e7ac80e9b70c75bf Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 19 Oct 2023 15:04:18 +0000 Subject: [PATCH 42/64] fix lint --- network/namespace.go | 4 ++++ network/namespace_windows.go | 2 +- network/secondary_endpoint_client_linux.go | 1 - 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/network/namespace.go b/network/namespace.go index ca505145b0..30be49d9f7 100644 --- a/network/namespace.go +++ b/network/namespace.go @@ -1,5 +1,9 @@ package network +import "errors" + +var errFileNotExist = errors.New("no such file or directory") + type NamespaceInterface interface { GetFd() uintptr GetName() string diff --git a/network/namespace_windows.go b/network/namespace_windows.go index c60641528d..6288066c53 100644 --- a/network/namespace_windows.go +++ b/network/namespace_windows.go @@ -7,7 +7,7 @@ import ( "errors" ) -const windowsImplErr = errors.New("windows impl err") +var windowsImplErr = errors.New("windows impl err") // Namespace represents a network namespace. type Namespace struct{} diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index d9c5e7fd67..8f63451b01 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -14,7 +14,6 @@ import ( var ( errorSecondaryEndpointClient = errors.New("SecondaryEndpointClient Error") - errFileNotExist = errors.New("no such file or directory") ) func newErrorSecondaryEndpointClient(err error) error { From 85e3eb6eb7cb19a3b70028a2c657fef8287ea2bd Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 20 Oct 2023 21:07:45 +0000 Subject: [PATCH 43/64] add comment --- network/secondary_endpoint_client_linux.go | 1 + 1 file changed, 1 insertion(+) diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index 8f63451b01..eea786da06 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -129,6 +129,7 @@ func (client *SecondaryEndpointClient) DeleteEndpoints(ep *endpoint) error { ns, err := client.nsClient.OpenNamespace(ep.NetworkNameSpace) if err != nil { if strings.Contains(err.Error(), errFileNotExist.Error()) { + // clear SecondaryInterfaces map since network namespace doesn't exist anymore ep.SecondaryInterfaces = make(map[string]*InterfaceInfo) return nil } From 89aa427624e0fa9337bc04eccd60fd11fa0d082e Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 20 Oct 2023 23:04:25 +0000 Subject: [PATCH 44/64] add extra delete endpoint test --- network/mock_namespace.go | 17 +++++++-- network/namespace_windows.go | 4 +-- network/secondary_endpoint_client_linux.go | 4 +-- network/secondary_endpoint_linux_test.go | 40 +++++++++++++++++++--- 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/network/mock_namespace.go b/network/mock_namespace.go index 1d7b558624..ec5be2c479 100644 --- a/network/mock_namespace.go +++ b/network/mock_namespace.go @@ -3,7 +3,15 @@ package network -type MockNamespace struct{} +import "github.com/pkg/errors" + +var errMockEnterNamespaceFailure = errors.New("failed to enter namespace") + +const failToEnterNamespaceName = "failns" + +type MockNamespace struct { + namespace string +} type MockNamespaceClient struct{} @@ -16,7 +24,7 @@ func (c *MockNamespaceClient) OpenNamespace(ns string) (NamespaceInterface, erro if ns == "" { return nil, errFileNotExist } - return &MockNamespace{}, nil + return &MockNamespace{namespace: ns}, nil } // GetCurrentThreadNamespace returns the caller thread's current namespace. @@ -35,11 +43,14 @@ func (ns *MockNamespace) GetFd() uintptr { } func (ns *MockNamespace) GetName() string { - return "nsname" + return ns.namespace } // Enter puts the caller thread inside the namespace. func (ns *MockNamespace) Enter() error { + if ns.namespace == failToEnterNamespaceName { + return errMockEnterNamespaceFailure + } return nil } diff --git a/network/namespace_windows.go b/network/namespace_windows.go index 6288066c53..19e3bbe3b9 100644 --- a/network/namespace_windows.go +++ b/network/namespace_windows.go @@ -7,7 +7,7 @@ import ( "errors" ) -var windowsImplErr = errors.New("windows impl err") +var errWindowsImpl = errors.New("windows impl err") // Namespace represents a network namespace. type Namespace struct{} @@ -20,7 +20,7 @@ func NewNamespaceClient() *NamespaceClient { // OpenNamespace creates a new namespace object for the given netns path. func (c *NamespaceClient) OpenNamespace(_ string) (NamespaceInterface, error) { - return nil, windowsImplErr + return nil, errWindowsImpl } // GetCurrentThreadNamespace returns the caller thread's current namespace. diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index eea786da06..edb23f40bb 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -12,9 +12,7 @@ import ( "go.uber.org/zap" ) -var ( - errorSecondaryEndpointClient = errors.New("SecondaryEndpointClient Error") -) +var errorSecondaryEndpointClient = errors.New("SecondaryEndpointClient Error") func newErrorSecondaryEndpointClient(err error) error { return errors.Wrapf(err, "%s", errorSecondaryEndpointClient) diff --git a/network/secondary_endpoint_linux_test.go b/network/secondary_endpoint_linux_test.go index 8a54894e49..f0ac46ddba 100644 --- a/network/secondary_endpoint_linux_test.go +++ b/network/secondary_endpoint_linux_test.go @@ -87,9 +87,10 @@ func TestSecondaryDeleteEndpoints(t *testing.T) { plc := platform.NewMockExecClient(false) tests := []struct { - name string - client *SecondaryEndpointClient - ep *endpoint + name string + client *SecondaryEndpointClient + ep *endpoint + wantErr bool }{ { name: "Delete endpoint happy path", @@ -136,14 +137,43 @@ func TestSecondaryDeleteEndpoints(t *testing.T) { }, }, }, + { + name: "Delete endpoint enter namespace failure", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), + }, + ep: &endpoint{ + NetworkNameSpace: failToEnterNamespaceName, + SecondaryInterfaces: map[string]*InterfaceInfo{ + "eth1": { + Name: "eth1", + Routes: []RouteInfo{ + { + Dst: net.IPNet{IP: net.ParseIP("192.168.0.4"), Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)}, + }, + }, + }, + }, + }, + wantErr: true, + }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { require.Len(t, tt.ep.SecondaryInterfaces, 1) - require.Nil(t, tt.client.DeleteEndpoints(tt.ep)) - require.Len(t, tt.ep.SecondaryInterfaces, 0) + if tt.wantErr { + require.Error(t, tt.client.DeleteEndpoints(tt.ep)) + require.Len(t, tt.ep.SecondaryInterfaces, 1) + } else { + require.Nil(t, tt.client.DeleteEndpoints(tt.ep)) + require.Len(t, tt.ep.SecondaryInterfaces, 0) + } }) } } From 0d1379840a61340b1e20762d15f0a7a5be17c589 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 23 Oct 2023 15:13:11 +0000 Subject: [PATCH 45/64] update test --- network/secondary_endpoint_client_linux.go | 7 +++++++ network/secondary_endpoint_linux_test.go | 24 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index edb23f40bb..a5f256ea83 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -1,6 +1,7 @@ package network import ( + "os" "strings" "github.com/Azure/azure-container-networking/netio" @@ -139,6 +140,11 @@ func (client *SecondaryEndpointClient) DeleteEndpoints(ep *endpoint) error { // Enter the container network namespace. logger.Info("Entering netns", zap.Any("NetNsPath", ep.NetworkNameSpace)) if err := ns.Enter(); err != nil { + if errors.Is(err, os.ErrNotExist) { + ep.SecondaryInterfaces = make(map[string]*InterfaceInfo) + return nil + } + return newErrorSecondaryEndpointClient(err) } @@ -153,6 +159,7 @@ func (client *SecondaryEndpointClient) DeleteEndpoints(ep *endpoint) error { for iface := range ep.SecondaryInterfaces { if err := client.netlink.SetLinkNetNs(iface, uintptr(vmns)); err != nil { logger.Error("Failed to move interface", zap.String("IfName", iface), zap.Error(newErrorSecondaryEndpointClient(err))) + continue } delete(ep.SecondaryInterfaces, iface) diff --git a/network/secondary_endpoint_linux_test.go b/network/secondary_endpoint_linux_test.go index f0ac46ddba..43494ffb86 100644 --- a/network/secondary_endpoint_linux_test.go +++ b/network/secondary_endpoint_linux_test.go @@ -161,6 +161,30 @@ func TestSecondaryDeleteEndpoints(t *testing.T) { }, wantErr: true, }, + { + name: "Delete endpoint netlink failure", + client: &SecondaryEndpointClient{ + netlink: netlink.NewMockNetlink(true, "netlink failure"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), + }, + ep: &endpoint{ + NetworkNameSpace: failToEnterNamespaceName, + SecondaryInterfaces: map[string]*InterfaceInfo{ + "eth1": { + Name: "eth1", + Routes: []RouteInfo{ + { + Dst: net.IPNet{IP: net.ParseIP("192.168.0.4"), Mask: net.CIDRMask(ipv4FullMask, ipv4Bits)}, + }, + }, + }, + }, + }, + wantErr: true, + }, } for _, tt := range tests { From f6984ad802a5636d662733271afe1af6fa35d614 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 8 Sep 2023 18:02:44 +0000 Subject: [PATCH 46/64] feat: update invokers to support swift 2 --- cni/network/invoker_azure.go | 23 +++++++++++++++ cni/network/invoker_azure_test.go | 33 +++++++++++++-------- cni/network/invoker_cns_test.go | 48 ------------------------------- 3 files changed, 44 insertions(+), 60 deletions(-) diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index f26d798f40..4bc98cbfec 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -58,7 +58,11 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er } // Call into IPAM plugin to allocate an address pool for the network. +<<<<<<< HEAD result, err := invoker.plugin.DelegateAdd(addConfig.nwCfg.IPAM.Type, addConfig.nwCfg) +======= + ipv4Result, err := invoker.plugin.DelegateAdd(addConfig.nwCfg.IPAM.Type, addConfig.nwCfg) +>>>>>>> 5267b700 (feat: update invokers to support swift 2) if err != nil && strings.Contains(err.Error(), ipam.ErrNoAvailableAddressPools.Error()) { invoker.deleteIpamState() logger.Info("Retry pool allocation after deleting IPAM state") @@ -69,14 +73,25 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er err = invoker.plugin.Errorf("Failed to allocate pool: %v", err) return addResult, err } +<<<<<<< HEAD if len(result.IPs) > 0 { addResult.hostSubnetPrefix = result.IPs[0].Address +======= + addResult.defaultCniResult = CNIResult{ipResult: ipv4Result, addressType: cns.Default, isDefaultInterface: true} + if len(ipv4Result.IPs) > 0 { + addResult.hostSubnetPrefix = ipv4Result.IPs[0].Address +>>>>>>> 5267b700 (feat: update invokers to support swift 2) } defer func() { if err != nil { +<<<<<<< HEAD if len(addResult.defaultInterfaceInfo.ipResult.IPs) > 0 { if er := invoker.Delete(&addResult.defaultInterfaceInfo.ipResult.IPs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { +======= + if len(addResult.defaultCniResult.ipResult.IPs) > 0 { + if er := invoker.Delete(&addResult.defaultCniResult.ipResult.IPs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { +>>>>>>> 5267b700 (feat: update invokers to support swift 2) err = invoker.plugin.Errorf("Failed to clean up IP's during Delete with error %v, after Add failed with error %w", er, err) } } else { @@ -100,14 +115,22 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er if err != nil { err = invoker.plugin.Errorf("Failed to allocate v6 pool: %v", err) } else { +<<<<<<< HEAD result.IPs = append(result.IPs, ipv6Result.IPs...) result.Routes = append(result.Routes, ipv6Result.Routes...) +======= + addResult.defaultCniResult.ipResult.IPs = append(addResult.defaultCniResult.ipResult.IPs, ipv6Result.IPs...) + addResult.defaultCniResult.ipResult.Routes = append(addResult.defaultCniResult.ipResult.Routes, ipv6Result.Routes...) +>>>>>>> 5267b700 (feat: update invokers to support swift 2) addResult.ipv6Enabled = true } } +<<<<<<< HEAD addResult.defaultInterfaceInfo = InterfaceInfo{ipResult: result, nicType: cns.InfraNIC} +======= +>>>>>>> 5267b700 (feat: update invokers to support swift 2) return addResult, err } diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index d000d899c6..1f5bed23fa 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -22,12 +22,19 @@ type mockDelegatePlugin struct { } type add struct { +<<<<<<< HEAD resultsIPv4 []*cniTypesCurr.Result resultsIPv6 []*cniTypesCurr.Result resultsIPv4Index int resultsIPv6Index int errv4 error errv6 error +======= + resultsIPv4 *cniTypesCurr.Result + resultsIPv6 *cniTypesCurr.Result + errv4 error + errv6 error +>>>>>>> 5267b700 (feat: update invokers to support swift 2) } func (d *add) DelegateAdd(pluginName string, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { @@ -35,23 +42,13 @@ func (d *add) DelegateAdd(pluginName string, nwCfg *cni.NetworkConfig) (*cniType if d.errv6 != nil { return nil, d.errv6 } - if d.resultsIPv6 == nil || d.resultsIPv6Index-1 > len(d.resultsIPv6) { - return nil, errors.New("no more ipv6 results in mock available") //nolint:goerr113 - } - res := d.resultsIPv6[d.resultsIPv6Index] - d.resultsIPv6Index++ - return res, nil + return d.resultsIPv6, nil } if d.errv4 != nil { return nil, d.errv4 } - if d.resultsIPv4 == nil || d.resultsIPv4Index-1 > len(d.resultsIPv4) { - return nil, errors.New("no more ipv4 results in mock available") //nolint:goerr113 - } - res := d.resultsIPv4[d.resultsIPv4Index] - d.resultsIPv4Index++ - return res, nil + return d.resultsIPv4, nil } type del struct { @@ -82,6 +79,7 @@ func getCIDRNotationForAddress(ipaddresswithcidr string) *net.IPNet { return ipnet } +<<<<<<< HEAD // getSingleResult returns an IPConfig with v4 or v6 IPNet func getSingleResult(ip string) []*cniTypesCurr.Result { return []*cniTypesCurr.Result{ @@ -92,6 +90,12 @@ func getSingleResult(ip string) []*cniTypesCurr.Result { }, }, }, +======= +func getResult(ips ...string) *cniTypesCurr.Result { + res := &cniTypesCurr.Result{} + for _, ip := range ips { + res.IPs = append(res.IPs, &cniTypesCurr.IPConfig{Address: *getCIDRNotationForAddress(ip)}) +>>>>>>> 5267b700 (feat: update invokers to support swift 2) } } @@ -231,8 +235,13 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { require.Nil(err) } +<<<<<<< HEAD fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ipamAddResult.defaultInterfaceInfo.ipResult) require.Exactly(tt.want, ipamAddResult.defaultInterfaceInfo.ipResult) +======= + fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ipamAddResult.defaultCniResult.ipResult) + require.Exactly(tt.want, ipamAddResult.defaultCniResult.ipResult) +>>>>>>> 5267b700 (feat: update invokers to support swift 2) }) } } diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 164805ba79..0b569e08ac 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -68,21 +68,12 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { } tests := []struct { -<<<<<<< HEAD name string fields fields args args wantDefaultResult *cniTypesCurr.Result wantSecondaryInterfacesInfo InterfaceInfo wantErr bool -======= - name string - fields fields - args args - wantDefaultResult *cniTypesCurr.Result - wantMultitenantResult *cniTypesCurr.Result - wantErr bool ->>>>>>> 5267b700 (feat: update invokers to support swift 2) }{ { name: "Test happy CNI Overlay add in v4overlay ipamMode", @@ -282,25 +273,16 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { PrimaryIP: "10.0.0.1", Subnet: "10.0.0.0/24", }, -<<<<<<< HEAD NICType: cns.InfraNIC, SkipDefaultRoutes: true, -======= - AddressType: cns.Default, ->>>>>>> 5267b700 (feat: update invokers to support swift 2) }, { PodIPConfig: cns.IPSubnet{ IPAddress: "20.240.1.242", PrefixLength: 24, }, -<<<<<<< HEAD NICType: cns.DelegatedVMNIC, MacAddress: macAddress, -======= - AddressType: cns.Secondary, - MacAddress: "12:34:56:78:9a:bc", ->>>>>>> 5267b700 (feat: update invokers to support swift 2) }, }, Response: cns.Response{ @@ -336,7 +318,6 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { }, }, }, -<<<<<<< HEAD wantSecondaryInterfacesInfo: InterfaceInfo{ ipResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ @@ -402,17 +383,6 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { }, }, err: nil, -======= - wantMultitenantResult: &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { - Address: *getCIDRNotationForAddress("20.240.1.242/24"), - }, - }, - Interfaces: []*cniTypesCurr.Interface{ - { - Mac: "12:34:56:78:9a:bc", ->>>>>>> 5267b700 (feat: update invokers to support swift 2) }, }, }, @@ -514,17 +484,10 @@ func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { require.NoError(err) } -<<<<<<< HEAD fmt.Printf("want:%+v\nrest:%+v\n", tt.wantSecondaryInterfacesInfo, ipamAddResult.secondaryInterfacesInfo) require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultInterfaceInfo.ipResult, "incorrect default response") if tt.wantSecondaryInterfacesInfo.ipResult != nil { require.EqualValues(tt.wantSecondaryInterfacesInfo, ipamAddResult.secondaryInterfacesInfo[0], "incorrect multitenant response") -======= - fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.cniResults) - require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultCniResult.ipResult, "incorrect default response") - if tt.wantMultitenantResult != nil { - require.Equalf(tt.wantMultitenantResult, ipamAddResult.cniResults[0].ipResult, "incorrect multitenant response") ->>>>>>> 5267b700 (feat: update invokers to support swift 2) } }) } @@ -747,17 +710,10 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { require.NoError(err) } -<<<<<<< HEAD fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.secondaryInterfacesInfo) require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultInterfaceInfo.ipResult, "incorrect default response") if tt.wantMultitenantResult != nil { require.Equalf(tt.wantMultitenantResult, ipamAddResult.secondaryInterfacesInfo[0].ipResult, "incorrect multitenant response") -======= - fmt.Printf("want:%+v\nrest:%+v\n", tt.wantMultitenantResult, ipamAddResult.cniResults) - require.Equalf(tt.wantDefaultResult, ipamAddResult.defaultCniResult.ipResult, "incorrect default response") - if tt.wantMultitenantResult != nil { - require.Equalf(tt.wantMultitenantResult, ipamAddResult.cniResults[0].ipResult, "incorrect multitenant response") ->>>>>>> 5267b700 (feat: update invokers to support swift 2) } }) } @@ -873,11 +829,7 @@ func TestCNSIPAMInvoker_Add_UnsupportedAPI(t *testing.T) { t.Fatalf("expected an error %+v but none received", err) } require.NoError(err) -<<<<<<< HEAD require.Equalf(tt.want, ipamAddResult.defaultInterfaceInfo.ipResult, "incorrect ipv4 response") -======= - require.Equalf(tt.want, ipamAddResult.defaultCniResult.ipResult, "incorrect ipv4 response") ->>>>>>> 5267b700 (feat: update invokers to support swift 2) }) } } From 9feaccbca6e9b47c3a2fa04669dda9ce58202f40 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Tue, 3 Oct 2023 17:02:31 +0000 Subject: [PATCH 47/64] address comments --- cni/network/invoker_azure.go | 23 --------------------- cni/network/invoker_azure_test.go | 34 +++++++++++-------------------- 2 files changed, 12 insertions(+), 45 deletions(-) diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index 4bc98cbfec..f26d798f40 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -58,11 +58,7 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er } // Call into IPAM plugin to allocate an address pool for the network. -<<<<<<< HEAD result, err := invoker.plugin.DelegateAdd(addConfig.nwCfg.IPAM.Type, addConfig.nwCfg) -======= - ipv4Result, err := invoker.plugin.DelegateAdd(addConfig.nwCfg.IPAM.Type, addConfig.nwCfg) ->>>>>>> 5267b700 (feat: update invokers to support swift 2) if err != nil && strings.Contains(err.Error(), ipam.ErrNoAvailableAddressPools.Error()) { invoker.deleteIpamState() logger.Info("Retry pool allocation after deleting IPAM state") @@ -73,25 +69,14 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er err = invoker.plugin.Errorf("Failed to allocate pool: %v", err) return addResult, err } -<<<<<<< HEAD if len(result.IPs) > 0 { addResult.hostSubnetPrefix = result.IPs[0].Address -======= - addResult.defaultCniResult = CNIResult{ipResult: ipv4Result, addressType: cns.Default, isDefaultInterface: true} - if len(ipv4Result.IPs) > 0 { - addResult.hostSubnetPrefix = ipv4Result.IPs[0].Address ->>>>>>> 5267b700 (feat: update invokers to support swift 2) } defer func() { if err != nil { -<<<<<<< HEAD if len(addResult.defaultInterfaceInfo.ipResult.IPs) > 0 { if er := invoker.Delete(&addResult.defaultInterfaceInfo.ipResult.IPs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { -======= - if len(addResult.defaultCniResult.ipResult.IPs) > 0 { - if er := invoker.Delete(&addResult.defaultCniResult.ipResult.IPs[0].Address, addConfig.nwCfg, nil, addConfig.options); er != nil { ->>>>>>> 5267b700 (feat: update invokers to support swift 2) err = invoker.plugin.Errorf("Failed to clean up IP's during Delete with error %v, after Add failed with error %w", er, err) } } else { @@ -115,22 +100,14 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er if err != nil { err = invoker.plugin.Errorf("Failed to allocate v6 pool: %v", err) } else { -<<<<<<< HEAD result.IPs = append(result.IPs, ipv6Result.IPs...) result.Routes = append(result.Routes, ipv6Result.Routes...) -======= - addResult.defaultCniResult.ipResult.IPs = append(addResult.defaultCniResult.ipResult.IPs, ipv6Result.IPs...) - addResult.defaultCniResult.ipResult.Routes = append(addResult.defaultCniResult.ipResult.Routes, ipv6Result.Routes...) ->>>>>>> 5267b700 (feat: update invokers to support swift 2) addResult.ipv6Enabled = true } } -<<<<<<< HEAD addResult.defaultInterfaceInfo = InterfaceInfo{ipResult: result, nicType: cns.InfraNIC} -======= ->>>>>>> 5267b700 (feat: update invokers to support swift 2) return addResult, err } diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index 1f5bed23fa..487db822bd 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -22,19 +22,12 @@ type mockDelegatePlugin struct { } type add struct { -<<<<<<< HEAD resultsIPv4 []*cniTypesCurr.Result resultsIPv6 []*cniTypesCurr.Result resultsIPv4Index int resultsIPv6Index int errv4 error errv6 error -======= - resultsIPv4 *cniTypesCurr.Result - resultsIPv6 *cniTypesCurr.Result - errv4 error - errv6 error ->>>>>>> 5267b700 (feat: update invokers to support swift 2) } func (d *add) DelegateAdd(pluginName string, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { @@ -42,13 +35,23 @@ func (d *add) DelegateAdd(pluginName string, nwCfg *cni.NetworkConfig) (*cniType if d.errv6 != nil { return nil, d.errv6 } - return d.resultsIPv6, nil + if d.resultsIPv6 == nil || d.resultsIPv6Index-1 > len(d.resultsIPv6) { + return nil, errors.New("no more ipv6 results in mock available") //nolint:goerr113 + } + res := d.resultsIPv6[d.resultsIPv6Index] + d.resultsIPv6Index++ + return res, nil } if d.errv4 != nil { return nil, d.errv4 } - return d.resultsIPv4, nil + if d.resultsIPv4 == nil || d.resultsIPv4Index-1 > len(d.resultsIPv4) { + return nil, errors.New("no more ipv4 results in mock available") //nolint:goerr113 + } + res := d.resultsIPv4[d.resultsIPv4Index] + d.resultsIPv4Index++ + return res, nil } type del struct { @@ -79,8 +82,6 @@ func getCIDRNotationForAddress(ipaddresswithcidr string) *net.IPNet { return ipnet } -<<<<<<< HEAD -// getSingleResult returns an IPConfig with v4 or v6 IPNet func getSingleResult(ip string) []*cniTypesCurr.Result { return []*cniTypesCurr.Result{ { @@ -90,12 +91,6 @@ func getSingleResult(ip string) []*cniTypesCurr.Result { }, }, }, -======= -func getResult(ips ...string) *cniTypesCurr.Result { - res := &cniTypesCurr.Result{} - for _, ip := range ips { - res.IPs = append(res.IPs, &cniTypesCurr.IPConfig{Address: *getCIDRNotationForAddress(ip)}) ->>>>>>> 5267b700 (feat: update invokers to support swift 2) } } @@ -235,13 +230,8 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { require.Nil(err) } -<<<<<<< HEAD fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ipamAddResult.defaultInterfaceInfo.ipResult) require.Exactly(tt.want, ipamAddResult.defaultInterfaceInfo.ipResult) -======= - fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ipamAddResult.defaultCniResult.ipResult) - require.Exactly(tt.want, ipamAddResult.defaultCniResult.ipResult) ->>>>>>> 5267b700 (feat: update invokers to support swift 2) }) } } From 19768792e65312f2f1d9cb1ebcb38d0d4f0d0025 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 22 Sep 2023 20:54:36 +0000 Subject: [PATCH 48/64] address comments --- network/secondary_endpoint_client_linux.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index a5f256ea83..a1146a95ad 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -94,6 +94,11 @@ func (client *SecondaryEndpointClient) SetupContainerInterfaces(epInfo *Endpoint } func (client *SecondaryEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { + log.Printf("[net] Setting link %v state up.", epInfo.IfName) + if err := client.netlink.SetLinkState(epInfo.IfName, true); err != nil { + return newErrorSecondaryEndpointClient(err) + } + if err := client.netUtilsClient.AssignIPToInterface(epInfo.IfName, epInfo.IPAddresses); err != nil { return newErrorSecondaryEndpointClient(err) } From 984cfe55a1a1dfa3f8761a0ec97f7839f5d3bb2c Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 8 Sep 2023 18:24:14 +0000 Subject: [PATCH 49/64] feat: refactor endpoint create/delete flow for swift 2 --- cni/network/network.go | 138 ++++++----- cni/network/network_test.go | 6 +- cni/network/network_windows.go | 2 +- cni/network/network_windows_test.go | 70 +++--- cnm/network/network.go | 2 +- network/endpoint.go | 7 +- network/endpoint_linux.go | 345 ++++++++++++++-------------- network/endpoint_test.go | 12 +- network/endpoint_windows.go | 8 +- network/manager.go | 8 +- network/manager_mock.go | 4 +- 11 files changed, 307 insertions(+), 295 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 09cc38792a..de12a8920e 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -362,23 +362,20 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { telemetry.SendCNIMetric(&cniMetric, plugin.tb) // Add Interfaces to result. - if ipamAddResult.ipv4Result == nil { - ipamAddResult.ipv4Result = &cniTypesCurr.Result{} + defaultCniResult := ipamAddResult.defaultCniResult.ipResult + if defaultCniResult == nil { + defaultCniResult = &cniTypesCurr.Result{} } iface := &cniTypesCurr.Interface{ Name: args.IfName, } - ipamAddResult.ipv4Result.Interfaces = append(ipamAddResult.ipv4Result.Interfaces, iface) + defaultCniResult.Interfaces = append(defaultCniResult.Interfaces, iface) - if ipamAddResult.ipv6Result != nil { - ipamAddResult.ipv4Result.IPs = append(ipamAddResult.ipv4Result.IPs, ipamAddResult.ipv6Result.IPs...) - } - - addSnatInterface(nwCfg, ipamAddResult.ipv4Result) + addSnatInterface(nwCfg, defaultCniResult) // Convert result to the requested CNI version. - res, vererr := ipamAddResult.ipv4Result.GetAsVersion(nwCfg.CNIVersion) + res, vererr := defaultCniResult.GetAsVersion(nwCfg.CNIVersion) if vererr != nil { logger.Error("GetAsVersion failed", zap.Error(vererr)) plugin.Error(vererr) @@ -391,7 +388,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { logger.Info("ADD command completed for", zap.String("pod", k8sPodName), - zap.Any("IPs", ipamAddResult.ipv4Result.IPs), + zap.Any("IPs", defaultCniResult.IPs), zap.Error(err)) }() @@ -424,7 +421,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { res, err = plugin.nnsClient.AddContainerNetworking(context.Background(), k8sPodName, args.Netns) if err == nil { - ipamAddResult.ipv4Result = convertNnsToCniResult(res, args.IfName, k8sPodName, "AddContainerNetworking") + ipamAddResult.defaultCniResult.ipResult = convertNnsToCniResult(res, args.IfName, k8sPodName, "AddContainerNetworking") } return err @@ -511,7 +508,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } if resultSecondAdd != nil { - ipamAddResult.ipv4Result = resultSecondAdd + ipamAddResult.defaultCniResult.ipResult = resultSecondAdd return nil } } @@ -533,12 +530,12 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if err != nil { return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) } - sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam:%+v v6:%+v", ipamAddResult.ipv4Result, ipamAddResult.ipv6Result)) + sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam:%+v", ipamAddResult.defaultCniResult)) } defer func() { //nolint:gocritic if err != nil { - plugin.cleanupAllocationOnError(ipamAddResult.ipv4Result, ipamAddResult.ipv6Result, nwCfg, args, options) + plugin.cleanupAllocationOnError(ipamAddResult.cniResults, nwCfg, args, options) } }() @@ -563,8 +560,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { createEndpointInternalOpt := createEndpointInternalOpt{ nwCfg: nwCfg, cnsNetworkConfig: ipamAddResult.ncResponse, - result: ipamAddResult.ipv4Result, - resultV6: ipamAddResult.ipv6Result, + ipamAddResult: ipamAddResult, azIpamResult: azIpamResult, args: args, nwInfo: &nwInfo, @@ -584,8 +580,10 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { return err } - sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded : IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", - ipamAddResult.ipv4Result.IPs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) + if ipamAddResult.defaultCniResult.ipResult != nil { + sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", + ipamAddResult.defaultCniResult.ipResult.IPs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) + } } return nil @@ -593,19 +591,16 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // cleanup allocated ipv4 and ipv6 addresses if they exist func (plugin *NetPlugin) cleanupAllocationOnError( - result, resultV6 *cniTypesCurr.Result, + cniResults []CNIResult, nwCfg *cni.NetworkConfig, args *cniSkel.CmdArgs, options map[string]interface{}, ) { - if result != nil && len(result.IPs) > 0 { - if er := plugin.ipamInvoker.Delete(&result.IPs[0].Address, nwCfg, args, options); er != nil { - logger.Error("Failed to cleanup ip allocation on failure", zap.Error(er)) - } - } - if resultV6 != nil && len(resultV6.IPs) > 0 { - if er := plugin.ipamInvoker.Delete(&resultV6.IPs[0].Address, nwCfg, args, options); er != nil { - logger.Error("Failed to cleanup ipv6 allocation on failure", zap.Error(er)) + for _, cniResult := range cniResults { + if cniResult.ipResult != nil && cniResult.addressType != cns.Secondary { + if er := plugin.ipamInvoker.Delete(&cniResult.ipResult.IPs[0].Address, nwCfg, args, options); er != nil { + logger.Error("Failed to cleanup ip allocation on failure", zap.Error(er)) + } } } } @@ -634,7 +629,7 @@ func (plugin *NetPlugin) createNetworkInternal( return nwInfo, err } - nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, ipamAddResult.ipv4Result) + nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, ipamAddResult.defaultCniResult.ipResult) if err != nil { err = plugin.Errorf("Failed to getDNSSettings: %v", err) return nwInfo, err @@ -658,7 +653,7 @@ func (plugin *NetPlugin) createNetworkInternal( IPV6Mode: ipamAddConfig.nwCfg.IPV6Mode, // TODO: check if IPV6Mode field can be deprecated IPAMType: ipamAddConfig.nwCfg.IPAM.Type, ServiceCidrs: ipamAddConfig.nwCfg.ServiceCidrs, - IsIPv6Enabled: ipamAddResult.ipv6Result != nil, + IsIPv6Enabled: ipamAddResult.ipv6Enabled, } if err = addSubnetToNetworkInfo(ipamAddResult, &nwInfo); err != nil { @@ -678,36 +673,22 @@ func (plugin *NetPlugin) createNetworkInternal( // construct network info with ipv4/ipv6 subnets func addSubnetToNetworkInfo(ipamAddResult IPAMAddResult, nwInfo *network.NetworkInfo) error { - var ( - podSubnetPrefix *net.IPNet - podSubnetV6Prefix *net.IPNet - ) - - _, podSubnetPrefix, err := net.ParseCIDR(ipamAddResult.ipv4Result.IPs[0].Address.String()) - if err != nil { - return fmt.Errorf("Failed to ParseCIDR for pod subnet prefix: %w", err) - } - - ipv4Subnet := network.SubnetInfo{ - Family: platform.AfINET, - Prefix: *podSubnetPrefix, - Gateway: ipamAddResult.ipv4Result.IPs[0].Gateway, - } - nwInfo.Subnets = append(nwInfo.Subnets, ipv4Subnet) - - // parse the ipv6 address and only add it to nwInfo if it's dual stack mode - if ipamAddResult.ipv6Result != nil && len(ipamAddResult.ipv6Result.IPs) > 0 { - _, podSubnetV6Prefix, err = net.ParseCIDR(ipamAddResult.ipv6Result.IPs[0].Address.String()) + for _, ipConfig := range ipamAddResult.defaultCniResult.ipResult.IPs { + ip, podSubnetPrefix, err := net.ParseCIDR(ipConfig.Address.String()) if err != nil { - return fmt.Errorf("Failed to ParseCIDR for pod subnet IPv6 prefix: %w", err) + return fmt.Errorf("Failed to ParseCIDR for pod subnet prefix: %w", err) } - ipv6Subnet := network.SubnetInfo{ - Family: platform.AfINET6, - Prefix: *podSubnetV6Prefix, - Gateway: ipamAddResult.ipv6Result.IPs[0].Gateway, + subnet := network.SubnetInfo{ + Family: platform.AfINET, + Prefix: *podSubnetPrefix, + Gateway: ipConfig.Gateway, } - nwInfo.Subnets = append(nwInfo.Subnets, ipv6Subnet) + if ip.To4() == nil { + subnet.Family = platform.AfINET6 + } + + nwInfo.Subnets = append(nwInfo.Subnets, subnet) } return nil @@ -716,8 +697,7 @@ func addSubnetToNetworkInfo(ipamAddResult IPAMAddResult, nwInfo *network.Network type createEndpointInternalOpt struct { nwCfg *cni.NetworkConfig cnsNetworkConfig *cns.GetNetworkContainerResponse - result *cniTypesCurr.Result - resultV6 *cniTypesCurr.Result + ipamAddResult IPAMAddResult azIpamResult *cniTypesCurr.Result args *cniSkel.CmdArgs nwInfo *network.NetworkInfo @@ -733,7 +713,8 @@ type createEndpointInternalOpt struct { func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) (network.EndpointInfo, error) { epInfo := network.EndpointInfo{} - epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, opt.result, opt.k8sNamespace) + defaultIPResult := opt.ipamAddResult.defaultCniResult.ipResult + epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, defaultIPResult, opt.k8sNamespace) if err != nil { err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) return epInfo, err @@ -741,7 +722,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) policyArgs := PolicyArgs{ nwInfo: opt.nwInfo, nwCfg: opt.nwCfg, - ipconfigs: opt.result.IPs, + ipconfigs: defaultIPResult.IPs, } endpointPolicies, err := getEndpointPolicies(policyArgs) if err != nil { @@ -779,32 +760,28 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) VnetCidrs: opt.nwCfg.VnetCidrs, ServiceCidrs: opt.nwCfg.ServiceCidrs, NATInfo: opt.natInfo, + AddressType: cns.Default, + IsDefaultInterface: opt.ipamAddResult.defaultCniResult.isDefaultInterface, } - isIPv6Enabled := opt.resultV6 != nil - epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, isIPv6Enabled) + epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) if err != nil { logger.Error("failed to get policies from runtime configurations", zap.Error(err)) return epInfo, plugin.Errorf(err.Error()) } - epInfo.Policies = append(epInfo.Policies, epPolicies...) // Populate addresses. - for _, ipconfig := range opt.result.IPs { + for _, ipconfig := range defaultIPResult.IPs { epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) } - if opt.resultV6 != nil { - // inject ipv6 routes to Linux pod + if opt.ipamAddResult.ipv6Enabled { epInfo.IPV6Mode = string(util.IpamMode(opt.nwCfg.IPAM.Mode)) // TODO: check IPV6Mode field can be deprecated and can we add IsIPv6Enabled flag for generic working - for _, ipconfig := range opt.resultV6.IPs { - epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) - } } // Populate routes. - for _, route := range opt.result.Routes { + for _, route := range defaultIPResult.Routes { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } @@ -813,7 +790,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) } if opt.nwCfg.MultiTenancy { - plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, opt.result) + plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, defaultIPResult) } setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) @@ -825,10 +802,29 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) return epInfo, plugin.Errorf(err.Error()) } + epInfos := []*network.EndpointInfo{&epInfo} + // get multitenant/secondary interface info + for _, multitenantResult := range opt.ipamAddResult.cniResults { + var addresses []net.IPNet + for _, ipconfig := range multitenantResult.ipResult.IPs { + addresses = append(addresses, ipconfig.Address) + } + + epInfos = append(epInfos, + &network.EndpointInfo{ + ContainerID: epInfo.ContainerID, + NetNsPath: epInfo.NetNsPath, + IPAddresses: addresses, + MacAddress: multitenantResult.macAddress, + AddressType: cns.Secondary, + IsDefaultInterface: multitenantResult.isDefaultInterface, + }) + } + // Create the endpoint. logger.Info("Creating endpoint", zap.String("endpointInfo", epInfo.PrettyString())) sendEvent(plugin, fmt.Sprintf("[cni-net] Creating endpoint %s.", epInfo.PrettyString())) - err = plugin.nm.CreateEndpoint(cnsclient, opt.nwInfo.Id, &epInfo) + err = plugin.nm.CreateEndpoint(cnsclient, opt.nwInfo.Id, epInfos) if err != nil { err = plugin.Errorf("Failed to create endpoint: %v", err) } diff --git a/cni/network/network_test.go b/cni/network/network_test.go index c53a61eabe..c75a79c830 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -1013,13 +1013,13 @@ func TestGetAllEndpointState(t *testing.T) { ep2 := getTestEndpoint("podname2", "podnamespace2", "10.0.0.2/24", "podinterfaceid2", "testcontainerid2") ep3 := getTestEndpoint("podname3", "podnamespace3", "10.240.1.242/16", "podinterfaceid3", "testcontainerid3") - err := plugin.nm.CreateEndpoint(nil, networkid, ep1) + err := plugin.nm.CreateEndpoint(nil, networkid, []*acnnetwork.EndpointInfo{ep1}) require.NoError(t, err) - err = plugin.nm.CreateEndpoint(nil, networkid, ep2) + err = plugin.nm.CreateEndpoint(nil, networkid, []*acnnetwork.EndpointInfo{ep2}) require.NoError(t, err) - err = plugin.nm.CreateEndpoint(nil, networkid, ep3) + err = plugin.nm.CreateEndpoint(nil, networkid, []*acnnetwork.EndpointInfo{ep3}) require.NoError(t, err) state, err := plugin.GetAllEndpointState(networkid) diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 8b27f9e4a4..3e9c0f3e22 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -161,7 +161,7 @@ func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResu // This will happen during ADD call if ipamAddResult != nil && ipamAddResult.ncResponse != nil { // networkName will look like ~ azure-vlan1-172-28-1-0_24 - ipAddrNet := ipamAddResult.ipv4Result.IPs[0].Address + ipAddrNet := ipamAddResult.defaultCniResult.ipResult.IPs[0].Address prefix, err := netip.ParsePrefix(ipAddrNet.String()) if err != nil { logger.Error("Error parsing network CIDR", diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index 2269b19486..04180ae0b1 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -367,12 +367,14 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ID: 1, }, }, - ipv4Result: &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP("10.240.0.5"), - Mask: net.CIDRMask(24, 32), + defaultCniResult: CNIResult{ + ipResult: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP("10.240.0.5"), + Mask: net.CIDRMask(24, 32), + }, }, }, }, @@ -402,12 +404,14 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ID: 1, }, }, - ipv4Result: &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP(""), - Mask: net.CIDRMask(24, 32), + defaultCniResult: CNIResult{ + ipResult: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP(""), + Mask: net.CIDRMask(24, 32), + }, }, }, }, @@ -437,12 +441,14 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ID: 1, }, }, - ipv4Result: &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP("10.0.00.6"), - Mask: net.CIDRMask(24, 32), + defaultCniResult: CNIResult{ + ipResult: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP("10.0.00.6"), + Mask: net.CIDRMask(24, 32), + }, }, }, }, @@ -472,12 +478,14 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ID: 1, }, }, - ipv4Result: &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP("10.0.0.6"), - Mask: net.CIDRMask(24, 32), + defaultCniResult: CNIResult{ + ipResult: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP("10.0.0.6"), + Mask: net.CIDRMask(24, 32), + }, }, }, }, @@ -503,12 +511,14 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }, ipamAddResult: &IPAMAddResult{ ncResponse: &cns.GetNetworkContainerResponse{}, - ipv4Result: &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { - Address: net.IPNet{ - IP: net.ParseIP("10.0.0.6"), - Mask: net.CIDRMask(24, 32), + defaultCniResult: CNIResult{ + ipResult: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: net.IPNet{ + IP: net.ParseIP("10.0.0.6"), + Mask: net.CIDRMask(24, 32), + }, }, }, }, diff --git a/cnm/network/network.go b/cnm/network/network.go index 0f10c5b94b..6ebc132511 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -246,7 +246,7 @@ func (plugin *netPlugin) createEndpoint(w http.ResponseWriter, r *http.Request) if err != nil { log.Errorf("failed to init CNS client", err) } - err = plugin.nm.CreateEndpoint(cnscli, req.NetworkID, &epInfo) + err = plugin.nm.CreateEndpoint(cnscli, req.NetworkID, []*network.EndpointInfo{&epInfo}) if err != nil { plugin.SendErrorResponse(w, err) return diff --git a/network/endpoint.go b/network/endpoint.go index 25d5c00871..4b45f59028 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -104,7 +104,6 @@ type RouteInfo struct { Table int } -// InterfaceInfo contains information for secondary interfaces type InterfaceInfo struct { Name string MacAddress net.HardwareAddr @@ -131,14 +130,14 @@ func (nw *network) newEndpoint( nl netlink.NetlinkInterface, plc platform.ExecClient, netioCli netio.NetIOInterface, - epInfo *EndpointInfo, + epInfo []*EndpointInfo, ) (*endpoint, error) { var ep *endpoint var err error defer func() { if err != nil { - logger.Error("Failed to create endpoint with err", zap.String("id", epInfo.Id), zap.Error(err)) + logger.Error("Failed to create endpoint with err", zap.String("id", epInfo[0].Id), zap.Error(err)) } }() @@ -149,7 +148,7 @@ func (nw *network) newEndpoint( return nil, err } - nw.Endpoints[epInfo.Id] = ep + nw.Endpoints[ep.Id] = ep logger.Info("Created endpoint. Num of endpoints", zap.Any("ep", ep), zap.Int("numEndpoints", len(nw.Endpoints))) return ep, nil } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 5a55aa520b..9cf3a908cc 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -10,6 +10,7 @@ import ( "net" "strings" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/networkutils" @@ -54,204 +55,203 @@ func (nw *network) newEndpointImpl( plc platform.ExecClient, netioCli netio.NetIOInterface, epClient EndpointClient, - epInfo *EndpointInfo, + epInfo []*EndpointInfo, ) (*endpoint, error) { - var containerIf *net.Interface - var ns *Namespace - var ep *endpoint - var err error - var hostIfName string - var contIfName string - var localIP string - var vlanid int = 0 - - if nw.Endpoints[epInfo.Id] != nil { - logger.Info("Endpoint alreday exists") - err = errEndpointExists - return nil, err + var ( + containerIf *net.Interface + err error + hostIfName string + contIfName string + localIP string + vlanid = 0 + ep = &endpoint{} + ) + if nw.extIf != nil { + ep.Gateways = []net.IP{nw.extIf.IPv4Gateway} } - if epInfo.Data != nil { - if _, ok := epInfo.Data[VlanIDKey]; ok { - vlanid = epInfo.Data[VlanIDKey].(int) - } - - if _, ok := epInfo.Data[LocalIPKey]; ok { - localIP = epInfo.Data[LocalIPKey].(string) - } - } + for _, epInfo := range epInfo { + epClient := epClient + switch epInfo.AddressType { + case cns.Secondary: + logger.Info("Secondary client") + if ep.SecondaryInterfaces == nil { + ep.SecondaryInterfaces = make(map[string]*InterfaceInfo) + } + if epClient == nil { + epClient = NewSecondaryEndpointClient(nl, plc, ep) + } + default: + if nw.Endpoints[epInfo.Id] != nil { + logger.Info("[net] Endpoint already exists.") + err = errEndpointExists + return nil, err + } - if _, ok := epInfo.Data[OptVethName]; ok { - key := epInfo.Data[OptVethName].(string) - logger.Info("Generate veth name based on the key provided", zap.String("key", key)) - vethname := generateVethName(key) - hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, vethname) - contIfName = fmt.Sprintf("%s%s2", hostVEthInterfacePrefix, vethname) - } else { - // Create a veth pair. - logger.Info("Generate veth name based on endpoint id") - hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, epInfo.Id[:7]) - contIfName = fmt.Sprintf("%s%s-2", hostVEthInterfacePrefix, epInfo.Id[:7]) - } + if epInfo.Data != nil { + if _, ok := epInfo.Data[VlanIDKey]; ok { + vlanid = epInfo.Data[VlanIDKey].(int) + } - // epClient is non-nil only when the endpoint is created for the unit test. - if epClient == nil { - //nolint:gocritic - if vlanid != 0 { - if nw.Mode == opModeTransparentVlan { - logger.Info("Transparent vlan client") - if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { - nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) + if _, ok := epInfo.Data[LocalIPKey]; ok { + localIP = epInfo.Data[LocalIPKey].(string) } - epClient = NewTransparentVlanEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc) + } + + if _, ok := epInfo.Data[OptVethName]; ok { + key := epInfo.Data[OptVethName].(string) + logger.Info("Generate veth name based on the key provided", zap.String("key", key)) + vethname := generateVethName(key) + hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, vethname) + contIfName = fmt.Sprintf("%s%s2", hostVEthInterfacePrefix, vethname) } else { - logger.Info("OVS client") - if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { - nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) - } + // Create a veth pair. + logger.Info("Generate veth name based on endpoint id") + hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, epInfo.Id[:7]) + contIfName = fmt.Sprintf("%s%s-2", hostVEthInterfacePrefix, epInfo.Id[:7]) + } - epClient = NewOVSEndpointClient( - nw, - epInfo, - hostIfName, - contIfName, - vlanid, - localIP, - nl, - ovsctl.NewOvsctl(), - plc) + ep.Id = epInfo.Id + ep.IfName = contIfName // container veth pair name. In cnm, we won't rename this and docker expects veth name. + ep.HostIfName = hostIfName + ep.InfraVnetIP = epInfo.InfraVnetIP + ep.LocalIP = localIP + ep.IPAddresses = epInfo.IPAddresses + ep.DNS = epInfo.DNS + ep.VlanID = vlanid + ep.EnableSnatOnHost = epInfo.EnableSnatOnHost + ep.EnableInfraVnet = epInfo.EnableInfraVnet + ep.EnableMultitenancy = epInfo.EnableMultiTenancy + ep.AllowInboundFromHostToNC = epInfo.AllowInboundFromHostToNC + ep.AllowInboundFromNCToHost = epInfo.AllowInboundFromNCToHost + ep.NetworkNameSpace = epInfo.NetNsPath + ep.ContainerID = epInfo.ContainerID + ep.PODName = epInfo.PODName + ep.PODNameSpace = epInfo.PODNameSpace + ep.Routes = epInfo.Routes + + // epClient is non-nil only when the endpoint is created for the unit test. + if epClient == nil { + //nolint:gocritic + if vlanid != 0 { + if nw.Mode == opModeTransparentVlan { + logger.Info("Transparent vlan client") + if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { + nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) + } + epClient = NewTransparentVlanEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc) + } else { + logger.Info("OVS client") + if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { + nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) + } + + epClient = NewOVSEndpointClient( + nw, + epInfo, + hostIfName, + contIfName, + vlanid, + localIP, + nl, + ovsctl.NewOvsctl(), + plc) + } + } else if nw.Mode != opModeTransparent { + logger.Info("Bridge client") + epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) + } else { + logger.Info("Transparent client") + epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) + } } - } else if nw.Mode != opModeTransparent { - logger.Info("Bridge client") - epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) - } else { - logger.Info("Transparent client") - epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) } - } - // Cleanup on failure. - defer func() { - if err != nil { - logger.Error("CNI error. Delete Endpoint and rules that are created", zap.Error(err), zap.String("contIfName", contIfName)) - endpt := &endpoint{ - Id: epInfo.Id, - IfName: contIfName, - HostIfName: hostIfName, - LocalIP: localIP, - IPAddresses: epInfo.IPAddresses, - DNS: epInfo.DNS, - VlanID: vlanid, - EnableSnatOnHost: epInfo.EnableSnatOnHost, - EnableMultitenancy: epInfo.EnableMultiTenancy, - AllowInboundFromHostToNC: epInfo.AllowInboundFromHostToNC, - AllowInboundFromNCToHost: epInfo.AllowInboundFromNCToHost, + //nolint:gocritic + defer func(client EndpointClient, contIfName string) { + // Cleanup on failure. + if err != nil { + logger.Error("CNI error. Delete Endpoint and rules that are created", zap.Error(err), zap.String("contIfName", contIfName)) + if containerIf != nil { + client.DeleteEndpointRules(ep) + } + // set deleteHostVeth to true to cleanup host veth interface if created + //nolint:errcheck // ignore error + client.DeleteEndpoints(ep) } + }(epClient, contIfName) - if containerIf != nil { - endpt.MacAddress = containerIf.HardwareAddr - epClient.DeleteEndpointRules(endpt) + err = func() error { + if epErr := epClient.AddEndpoints(epInfo); epErr != nil { + return epErr } - if nw.extIf != nil { - endpt.Gateways = []net.IP{nw.extIf.IPv4Gateway} + if epInfo.AddressType == cns.Default { + var epErr error + containerIf, epErr = netioCli.GetNetworkInterfaceByName(contIfName) + if epErr != nil { + return epErr + } + ep.MacAddress = containerIf.HardwareAddr } - // set deleteHostVeth to true to cleanup host veth interface if created - //nolint:errcheck // ignore error - epClient.DeleteEndpoints(endpt) - } - }() - - if err = epClient.AddEndpoints(epInfo); err != nil { - return nil, err - } - containerIf, err = netioCli.GetNetworkInterfaceByName(contIfName) - if err != nil { - return nil, err - } + // Setup rules for IP addresses on the container interface. + if epErr := epClient.AddEndpointRules(epInfo); epErr != nil { + return epErr + } - // Setup rules for IP addresses on the container interface. - if err = epClient.AddEndpointRules(epInfo); err != nil { - return nil, err - } + // If a network namespace for the container interface is specified... + if epInfo.NetNsPath != "" { + // Open the network namespace. + logger.Info("Opening netns", zap.Any("NetNsPath", epInfo.NetNsPath)) + ns, epErr := OpenNamespace(epInfo.NetNsPath) + if epErr != nil { + return epErr + } + defer ns.Close() - // If a network namespace for the container interface is specified... - if epInfo.NetNsPath != "" { - // Open the network namespace. - logger.Info("Opening netns", zap.Any("NetNsPath", epInfo.NetNsPath)) - ns, err = OpenNamespace(epInfo.NetNsPath) - if err != nil { - return nil, err - } - defer ns.Close() + if epErr := epClient.MoveEndpointsToContainerNS(epInfo, ns.GetFd()); epErr != nil { + return epErr + } - if err := epClient.MoveEndpointsToContainerNS(epInfo, ns.GetFd()); err != nil { - return nil, err - } + // Enter the container network namespace. + logger.Info("Entering netns", zap.Any("NetNsPath", epInfo.NetNsPath)) + if epErr := ns.Enter(); epErr != nil { + return epErr + } - // Enter the container network namespace. - logger.Info("Entering netns", zap.Any("NetNsPath", epInfo.NetNsPath)) - if err = ns.Enter(); err != nil { - return nil, err - } + // Return to host network namespace. + defer func() { + logger.Info("Exiting netns", zap.Any("NetNsPath", epInfo.NetNsPath)) + if epErr := ns.Exit(); epErr != nil { + logger.Error("Failed to exit netns with", zap.Error(err)) + } + }() + } - // Return to host network namespace. - defer func() { - logger.Info("Exiting netns", zap.Any("NetNsPath", epInfo.NetNsPath)) - if err := ns.Exit(); err != nil { - logger.Error("Failed to exit netns with", zap.Error(err)) + if epInfo.IPV6Mode != "" { + // Enable ipv6 setting in container + logger.Info("Enable ipv6 setting in container.") + nuc := networkutils.NewNetworkUtils(nl, plc) + if epErr := nuc.UpdateIPV6Setting(0); epErr != nil { + return fmt.Errorf("Enable ipv6 in container failed:%w", epErr) + } } - }() - } - if epInfo.IPV6Mode != "" { - // Enable ipv6 setting in container - logger.Info("Enable ipv6 setting in container.") - nuc := networkutils.NewNetworkUtils(nl, plc) - if err = nuc.UpdateIPV6Setting(0); err != nil { - return nil, fmt.Errorf("Enable ipv6 in container failed:%w", err) - } - } + // If a name for the container interface is specified... + if epInfo.IfName != "" { + if epErr := epClient.SetupContainerInterfaces(epInfo); epErr != nil { + return epErr + } + } - // If a name for the container interface is specified... - if epInfo.IfName != "" { - if err = epClient.SetupContainerInterfaces(epInfo); err != nil { + return epClient.ConfigureContainerInterfacesAndRoutes(epInfo) + }() + if err != nil { return nil, err } } - if err = epClient.ConfigureContainerInterfacesAndRoutes(epInfo); err != nil { - return nil, err - } - - // Create the endpoint object. - ep = &endpoint{ - Id: epInfo.Id, - IfName: contIfName, // container veth pair name. In cnm, we won't rename this and docker expects veth name. - HostIfName: hostIfName, - MacAddress: containerIf.HardwareAddr, - InfraVnetIP: epInfo.InfraVnetIP, - LocalIP: localIP, - IPAddresses: epInfo.IPAddresses, - DNS: epInfo.DNS, - VlanID: vlanid, - EnableSnatOnHost: epInfo.EnableSnatOnHost, - EnableInfraVnet: epInfo.EnableInfraVnet, - EnableMultitenancy: epInfo.EnableMultiTenancy, - AllowInboundFromHostToNC: epInfo.AllowInboundFromHostToNC, - AllowInboundFromNCToHost: epInfo.AllowInboundFromNCToHost, - NetworkNameSpace: epInfo.NetNsPath, - ContainerID: epInfo.ContainerID, - PODName: epInfo.PODName, - PODNameSpace: epInfo.PODNameSpace, - } - - if nw.extIf != nil { - ep.Gateways = []net.IP{nw.extIf.IPv4Gateway} - } - - ep.Routes = append(ep.Routes, epInfo.Routes...) return ep, nil } @@ -276,6 +276,13 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. } else if nw.Mode != opModeTransparent { epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl, plc) } else { + if len(ep.SecondaryInterfaces) > 0 { + epClient = NewSecondaryEndpointClient(nl, plc, ep) + epClient.DeleteEndpointRules(ep) + //nolint:errcheck // ignore error + epClient.DeleteEndpoints(ep) + } + epClient = NewTransparentEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl, plc) } } diff --git a/network/endpoint_test.go b/network/endpoint_test.go index 5ad1e7ceef..16a0e71d0f 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -181,7 +181,7 @@ var _ = Describe("Test Endpoint", func() { It("Should be added", func() { // Add endpoint with valid id ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(false), epInfo) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(false), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -193,7 +193,7 @@ var _ = Describe("Test Endpoint", func() { extIf: &externalInterface{IPv4Gateway: net.ParseIP("192.168.0.1")}, } ep, err := nw2.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(false), epInfo) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(false), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -209,7 +209,7 @@ var _ = Describe("Test Endpoint", func() { Expect(err).ToNot(HaveOccurred()) // Adding endpoint with same id should fail and delete should cleanup the state ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), mockCli, epInfo) + netio.NewMockNetIO(false, 0), mockCli, []*EndpointInfo{epInfo}) Expect(err).To(HaveOccurred()) Expect(ep2).To(BeNil()) assert.Contains(GinkgoT(), err.Error(), "Endpoint already exists") @@ -219,7 +219,7 @@ var _ = Describe("Test Endpoint", func() { // Adding an endpoint with an id. mockCli := NewMockEndpointClient(false) ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), mockCli, epInfo) + netio.NewMockNetIO(false, 0), mockCli, []*EndpointInfo{epInfo}) Expect(err).ToNot(HaveOccurred()) Expect(ep2).ToNot(BeNil()) Expect(len(mockCli.endpoints)).To(Equal(1)) @@ -242,11 +242,11 @@ var _ = Describe("Test Endpoint", func() { Id: "768e8deb-eth1", } ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(true), epInfo) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(true), []*EndpointInfo{epInfo}) Expect(err).To(HaveOccurred()) Expect(ep).To(BeNil()) ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(false), epInfo) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(false), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 5e66f02e51..cd1d6e5a3a 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -64,16 +64,16 @@ func ConstructEndpointID(containerID string, netNsPath string, ifName string) (s } // newEndpointImpl creates a new endpoint in the network. -func (nw *network) newEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, plc platform.ExecClient, _ netio.NetIOInterface, _ EndpointClient, epInfo *EndpointInfo) (*endpoint, error) { - if useHnsV2, err := UseHnsV2(epInfo.NetNsPath); useHnsV2 { +func (nw *network) newEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, plc platform.ExecClient, _ netio.NetIOInterface, _ EndpointClient, epInfo []*EndpointInfo) (*endpoint, error) { + if useHnsV2, err := UseHnsV2(epInfo[0].NetNsPath); useHnsV2 { if err != nil { return nil, err } - return nw.newEndpointImplHnsV2(cli, epInfo) + return nw.newEndpointImplHnsV2(cli, epInfo[0]) } - return nw.newEndpointImplHnsV1(epInfo, plc) + return nw.newEndpointImplHnsV1(epInfo[0], plc) } // newEndpointImplHnsV1 creates a new endpoint in the network using HnsV1 diff --git a/network/manager.go b/network/manager.go index ab849c307a..1b6bd3fd61 100644 --- a/network/manager.go +++ b/network/manager.go @@ -85,7 +85,7 @@ type NetworkManager interface { FindNetworkIDFromNetNs(netNs string) (string, error) GetNumEndpointsByContainerID(containerID string) int - CreateEndpoint(client apipaClient, networkID string, epInfo *EndpointInfo) error + CreateEndpoint(client apipaClient, networkID string, epInfo []*EndpointInfo) error DeleteEndpoint(networkID string, endpointID string) error GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) GetAllEndpoints(networkID string) (map[string]*EndpointInfo, error) @@ -327,7 +327,7 @@ func (nm *networkManager) GetNetworkInfo(networkId string) (NetworkInfo, error) } // CreateEndpoint creates a new container endpoint. -func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epInfo *EndpointInfo) error { +func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epInfo []*EndpointInfo) error { nm.Lock() defer nm.Unlock() @@ -337,9 +337,9 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn } if nw.VlanId != 0 { - if epInfo.Data[VlanIDKey] == nil { + if epInfo[0].Data[VlanIDKey] == nil { logger.Info("overriding endpoint vlanid with network vlanid") - epInfo.Data[VlanIDKey] = nw.VlanId + epInfo[0].Data[VlanIDKey] = nw.VlanId } } diff --git a/network/manager_mock.go b/network/manager_mock.go index 8b8f4af311..a77623e5d6 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -52,8 +52,8 @@ func (nm *MockNetworkManager) GetNetworkInfo(networkID string) (NetworkInfo, err } // CreateEndpoint mock -func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, networkID string, epInfo *EndpointInfo) error { - nm.TestEndpointInfoMap[epInfo.Id] = epInfo +func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfo []*EndpointInfo) error { + nm.TestEndpointInfoMap[epInfo[0].Id] = epInfo[0] return nil } From 9c5406c95f72a7b491f2d223dd151d278b3b2839 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 18 Sep 2023 22:10:24 +0000 Subject: [PATCH 50/64] address comments --- cni/network/network.go | 18 ++-- network/endpoint_linux.go | 189 ++++++++++++++++++------------------ network/endpoint_windows.go | 1 + network/manager.go | 1 + 4 files changed, 102 insertions(+), 107 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index de12a8920e..71df50f503 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -580,10 +580,8 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { return err } - if ipamAddResult.defaultCniResult.ipResult != nil { - sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", - ipamAddResult.defaultCniResult.ipResult.IPs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) - } + sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", + ipamAddResult.defaultCniResult.ipResult.IPs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) } return nil @@ -597,7 +595,7 @@ func (plugin *NetPlugin) cleanupAllocationOnError( options map[string]interface{}, ) { for _, cniResult := range cniResults { - if cniResult.ipResult != nil && cniResult.addressType != cns.Secondary { + if cniResult.ipResult != nil && cniResult.addressType == cns.Default { if er := plugin.ipamInvoker.Delete(&cniResult.ipResult.IPs[0].Address, nwCfg, args, options); er != nil { logger.Error("Failed to cleanup ip allocation on failure", zap.Error(er)) } @@ -803,10 +801,10 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) } epInfos := []*network.EndpointInfo{&epInfo} - // get multitenant/secondary interface info - for _, multitenantResult := range opt.ipamAddResult.cniResults { + // get secondary interface info + for _, secondaryCniResult := range opt.ipamAddResult.cniResults { var addresses []net.IPNet - for _, ipconfig := range multitenantResult.ipResult.IPs { + for _, ipconfig := range secondaryCniResult.ipResult.IPs { addresses = append(addresses, ipconfig.Address) } @@ -815,9 +813,9 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) ContainerID: epInfo.ContainerID, NetNsPath: epInfo.NetNsPath, IPAddresses: addresses, - MacAddress: multitenantResult.macAddress, + MacAddress: secondaryCniResult.macAddress, AddressType: cns.Secondary, - IsDefaultInterface: multitenantResult.isDefaultInterface, + IsDefaultInterface: secondaryCniResult.isDefaultInterface, }) } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 9cf3a908cc..27475eb67b 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -54,116 +54,111 @@ func (nw *network) newEndpointImpl( nl netlink.NetlinkInterface, plc platform.ExecClient, netioCli netio.NetIOInterface, - epClient EndpointClient, + testEpClient EndpointClient, epInfo []*EndpointInfo, ) (*endpoint, error) { var ( - containerIf *net.Interface - err error - hostIfName string - contIfName string - localIP string - vlanid = 0 - ep = &endpoint{} + containerIf *net.Interface + err error + hostIfName string + contIfName string + localIP string + vlanid = 0 + defaultEpInfo = epInfo[0] ) - if nw.extIf != nil { - ep.Gateways = []net.IP{nw.extIf.IPv4Gateway} + + if nw.Endpoints[defaultEpInfo.Id] != nil { + logger.Info("[net] Endpoint already exists.") + err = errEndpointExists + return nil, err } - for _, epInfo := range epInfo { - epClient := epClient - switch epInfo.AddressType { - case cns.Secondary: - logger.Info("Secondary client") - if ep.SecondaryInterfaces == nil { - ep.SecondaryInterfaces = make(map[string]*InterfaceInfo) - } - if epClient == nil { - epClient = NewSecondaryEndpointClient(nl, plc, ep) - } - default: - if nw.Endpoints[epInfo.Id] != nil { - logger.Info("[net] Endpoint already exists.") - err = errEndpointExists - return nil, err - } + if defaultEpInfo.Data != nil { + if _, ok := defaultEpInfo.Data[VlanIDKey]; ok { + vlanid = defaultEpInfo.Data[VlanIDKey].(int) + } - if epInfo.Data != nil { - if _, ok := epInfo.Data[VlanIDKey]; ok { - vlanid = epInfo.Data[VlanIDKey].(int) - } + if _, ok := defaultEpInfo.Data[LocalIPKey]; ok { + localIP = defaultEpInfo.Data[LocalIPKey].(string) + } + } - if _, ok := epInfo.Data[LocalIPKey]; ok { - localIP = epInfo.Data[LocalIPKey].(string) - } - } + if _, ok := defaultEpInfo.Data[OptVethName]; ok { + key := defaultEpInfo.Data[OptVethName].(string) + logger.Info("Generate veth name based on the key provided", zap.String("key", key)) + vethname := generateVethName(key) + hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, vethname) + contIfName = fmt.Sprintf("%s%s2", hostVEthInterfacePrefix, vethname) + } else { + // Create a veth pair. + logger.Info("Generate veth name based on endpoint id") + hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, defaultEpInfo.Id[:7]) + contIfName = fmt.Sprintf("%s%s-2", hostVEthInterfacePrefix, defaultEpInfo.Id[:7]) + } - if _, ok := epInfo.Data[OptVethName]; ok { - key := epInfo.Data[OptVethName].(string) - logger.Info("Generate veth name based on the key provided", zap.String("key", key)) - vethname := generateVethName(key) - hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, vethname) - contIfName = fmt.Sprintf("%s%s2", hostVEthInterfacePrefix, vethname) - } else { - // Create a veth pair. - logger.Info("Generate veth name based on endpoint id") - hostIfName = fmt.Sprintf("%s%s", hostVEthInterfacePrefix, epInfo.Id[:7]) - contIfName = fmt.Sprintf("%s%s-2", hostVEthInterfacePrefix, epInfo.Id[:7]) - } + ep := &endpoint{ + Id: defaultEpInfo.Id, + IfName: contIfName, // container veth pair name. In cnm, we won't rename this and docker expects veth name. + HostIfName: hostIfName, + InfraVnetIP: defaultEpInfo.InfraVnetIP, + LocalIP: localIP, + IPAddresses: defaultEpInfo.IPAddresses, + DNS: defaultEpInfo.DNS, + VlanID: vlanid, + EnableSnatOnHost: defaultEpInfo.EnableSnatOnHost, + EnableInfraVnet: defaultEpInfo.EnableInfraVnet, + EnableMultitenancy: defaultEpInfo.EnableMultiTenancy, + AllowInboundFromHostToNC: defaultEpInfo.AllowInboundFromHostToNC, + AllowInboundFromNCToHost: defaultEpInfo.AllowInboundFromNCToHost, + NetworkNameSpace: defaultEpInfo.NetNsPath, + ContainerID: defaultEpInfo.ContainerID, + PODName: defaultEpInfo.PODName, + PODNameSpace: defaultEpInfo.PODNameSpace, + Routes: defaultEpInfo.Routes, + SecondaryInterfaces: make(map[string]*InterfaceInfo), + } + if nw.extIf != nil { + ep.Gateways = []net.IP{nw.extIf.IPv4Gateway} + } - ep.Id = epInfo.Id - ep.IfName = contIfName // container veth pair name. In cnm, we won't rename this and docker expects veth name. - ep.HostIfName = hostIfName - ep.InfraVnetIP = epInfo.InfraVnetIP - ep.LocalIP = localIP - ep.IPAddresses = epInfo.IPAddresses - ep.DNS = epInfo.DNS - ep.VlanID = vlanid - ep.EnableSnatOnHost = epInfo.EnableSnatOnHost - ep.EnableInfraVnet = epInfo.EnableInfraVnet - ep.EnableMultitenancy = epInfo.EnableMultiTenancy - ep.AllowInboundFromHostToNC = epInfo.AllowInboundFromHostToNC - ep.AllowInboundFromNCToHost = epInfo.AllowInboundFromNCToHost - ep.NetworkNameSpace = epInfo.NetNsPath - ep.ContainerID = epInfo.ContainerID - ep.PODName = epInfo.PODName - ep.PODNameSpace = epInfo.PODNameSpace - ep.Routes = epInfo.Routes - - // epClient is non-nil only when the endpoint is created for the unit test. - if epClient == nil { - //nolint:gocritic - if vlanid != 0 { - if nw.Mode == opModeTransparentVlan { - logger.Info("Transparent vlan client") - if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { - nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) - } - epClient = NewTransparentVlanEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc) - } else { - logger.Info("OVS client") - if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { - nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) - } - - epClient = NewOVSEndpointClient( - nw, - epInfo, - hostIfName, - contIfName, - vlanid, - localIP, - nl, - ovsctl.NewOvsctl(), - plc) + for _, epInfo := range epInfo { + // testEpClient is non-nil only when the endpoint is created for the unit test + epClient := testEpClient + if epClient == nil { + //nolint:gocritic + if vlanid != 0 { + if nw.Mode == opModeTransparentVlan { + logger.Info("Transparent vlan client") + if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { + nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) } - } else if nw.Mode != opModeTransparent { - logger.Info("Bridge client") - epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) + epClient = NewTransparentVlanEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc) } else { - logger.Info("Transparent client") - epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) + logger.Info("OVS client") + if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { + nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) + } + + epClient = NewOVSEndpointClient( + nw, + epInfo, + hostIfName, + contIfName, + vlanid, + localIP, + nl, + ovsctl.NewOvsctl(), + plc) } + } else if nw.Mode != opModeTransparent { + logger.Info("Bridge client") + epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) + } else if epInfo.AddressType == cns.Secondary { + logger.Info("Secondary client") + epClient = NewSecondaryEndpointClient(nl, plc, ep) + } else { + logger.Info("Transparent client") + epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) } } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index cd1d6e5a3a..42c8e7af14 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -65,6 +65,7 @@ func ConstructEndpointID(containerID string, netNsPath string, ifName string) (s // newEndpointImpl creates a new endpoint in the network. func (nw *network) newEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, plc platform.ExecClient, _ netio.NetIOInterface, _ EndpointClient, epInfo []*EndpointInfo) (*endpoint, error) { + // there is only 1 epInfo for windows, multiple interfaces will be added in the future if useHnsV2, err := UseHnsV2(epInfo[0].NetNsPath); useHnsV2 { if err != nil { return nil, err diff --git a/network/manager.go b/network/manager.go index 1b6bd3fd61..2bda9f23a2 100644 --- a/network/manager.go +++ b/network/manager.go @@ -337,6 +337,7 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn } if nw.VlanId != 0 { + // the first entry in epInfo is default address type if epInfo[0].Data[VlanIDKey] == nil { logger.Info("overriding endpoint vlanid with network vlanid") epInfo[0].Data[VlanIDKey] = nw.VlanId From 165cacd475aa8458328bc2bb9e0b00b2756c79e1 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 22 Sep 2023 21:07:31 +0000 Subject: [PATCH 51/64] address comments --- cni/network/network.go | 6 +++--- network/endpoint_linux.go | 4 ++-- network/secondary_endpoint_client_linux.go | 5 ----- network/secondary_endpoint_linux_test.go | 2 +- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 71df50f503..8a87508d46 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -595,7 +595,7 @@ func (plugin *NetPlugin) cleanupAllocationOnError( options map[string]interface{}, ) { for _, cniResult := range cniResults { - if cniResult.ipResult != nil && cniResult.addressType == cns.Default { + if cniResult.ipResult != nil && cniResult.nicType == cns.Default { if er := plugin.ipamInvoker.Delete(&cniResult.ipResult.IPs[0].Address, nwCfg, args, options); er != nil { logger.Error("Failed to cleanup ip allocation on failure", zap.Error(er)) } @@ -758,7 +758,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) VnetCidrs: opt.nwCfg.VnetCidrs, ServiceCidrs: opt.nwCfg.ServiceCidrs, NATInfo: opt.natInfo, - AddressType: cns.Default, + NICType: cns.Default, IsDefaultInterface: opt.ipamAddResult.defaultCniResult.isDefaultInterface, } @@ -814,7 +814,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) NetNsPath: epInfo.NetNsPath, IPAddresses: addresses, MacAddress: secondaryCniResult.macAddress, - AddressType: cns.Secondary, + NICType: cns.Secondary, IsDefaultInterface: secondaryCniResult.isDefaultInterface, }) } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 27475eb67b..3ebf867dec 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -153,7 +153,7 @@ func (nw *network) newEndpointImpl( } else if nw.Mode != opModeTransparent { logger.Info("Bridge client") epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) - } else if epInfo.AddressType == cns.Secondary { + } else if epInfo.NICType == cns.Secondary { logger.Info("Secondary client") epClient = NewSecondaryEndpointClient(nl, plc, ep) } else { @@ -181,7 +181,7 @@ func (nw *network) newEndpointImpl( return epErr } - if epInfo.AddressType == cns.Default { + if epInfo.NICType == cns.Default { var epErr error containerIf, epErr = netioCli.GetNetworkInterfaceByName(contIfName) if epErr != nil { diff --git a/network/secondary_endpoint_client_linux.go b/network/secondary_endpoint_client_linux.go index a1146a95ad..a5f256ea83 100644 --- a/network/secondary_endpoint_client_linux.go +++ b/network/secondary_endpoint_client_linux.go @@ -94,11 +94,6 @@ func (client *SecondaryEndpointClient) SetupContainerInterfaces(epInfo *Endpoint } func (client *SecondaryEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - log.Printf("[net] Setting link %v state up.", epInfo.IfName) - if err := client.netlink.SetLinkState(epInfo.IfName, true); err != nil { - return newErrorSecondaryEndpointClient(err) - } - if err := client.netUtilsClient.AssignIPToInterface(epInfo.IfName, epInfo.IPAddresses); err != nil { return newErrorSecondaryEndpointClient(err) } diff --git a/network/secondary_endpoint_linux_test.go b/network/secondary_endpoint_linux_test.go index 43494ffb86..775ffeeb6a 100644 --- a/network/secondary_endpoint_linux_test.go +++ b/network/secondary_endpoint_linux_test.go @@ -265,7 +265,7 @@ func TestSecondaryConfigureContainerInterfacesAndRoutes(t *testing.T) { netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 2), + netioshim: netio.NewMockNetIO(true, 1), ep: &endpoint{SecondaryInterfaces: map[string]*InterfaceInfo{"eth1": {Name: "eth1"}}}, }, epInfo: &EndpointInfo{ From f1cb36ce91799aa57d66c705e0561a78b3b3c2a5 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 25 Sep 2023 17:41:52 +0000 Subject: [PATCH 52/64] address linter --- network/endpoint_linux.go | 1 + 1 file changed, 1 insertion(+) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 3ebf867dec..ffddfd539b 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -176,6 +176,7 @@ func (nw *network) newEndpointImpl( } }(epClient, contIfName) + //nolint:wrapcheck // ignore wrap check err = func() error { if epErr := epClient.AddEndpoints(epInfo); epErr != nil { return epErr From f9d7c19094d4b6c95a8cccb00db5dae1143f41c7 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 25 Sep 2023 21:01:42 +0000 Subject: [PATCH 53/64] update based on contract changes --- cni/network/network.go | 44 +++++++++++++++++----------------- cni/network/network_windows.go | 2 +- network/endpoint_linux.go | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 8a87508d46..f930a9c3df 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -362,7 +362,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { telemetry.SendCNIMetric(&cniMetric, plugin.tb) // Add Interfaces to result. - defaultCniResult := ipamAddResult.defaultCniResult.ipResult + defaultCniResult := ipamAddResult.defaultInterfaceInfo.ipResult if defaultCniResult == nil { defaultCniResult = &cniTypesCurr.Result{} } @@ -421,7 +421,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { res, err = plugin.nnsClient.AddContainerNetworking(context.Background(), k8sPodName, args.Netns) if err == nil { - ipamAddResult.defaultCniResult.ipResult = convertNnsToCniResult(res, args.IfName, k8sPodName, "AddContainerNetworking") + ipamAddResult.defaultInterfaceInfo.ipResult = convertNnsToCniResult(res, args.IfName, k8sPodName, "AddContainerNetworking") } return err @@ -508,7 +508,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } if resultSecondAdd != nil { - ipamAddResult.defaultCniResult.ipResult = resultSecondAdd + ipamAddResult.defaultInterfaceInfo.ipResult = resultSecondAdd return nil } } @@ -530,12 +530,12 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if err != nil { return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) } - sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam:%+v", ipamAddResult.defaultCniResult)) + sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam:%+v", ipamAddResult.defaultInterfaceInfo)) } defer func() { //nolint:gocritic if err != nil { - plugin.cleanupAllocationOnError(ipamAddResult.cniResults, nwCfg, args, options) + plugin.cleanupAllocationOnError(ipamAddResult.secondaryInterfaceInfo, nwCfg, args, options) } }() @@ -581,7 +581,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } sendEvent(plugin, fmt.Sprintf("CNI ADD succeeded: IP:%+v, VlanID: %v, podname %v, namespace %v numendpoints:%d", - ipamAddResult.defaultCniResult.ipResult.IPs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) + ipamAddResult.defaultInterfaceInfo.ipResult.IPs, epInfo.Data[network.VlanIDKey], k8sPodName, k8sNamespace, plugin.nm.GetNumberOfEndpoints("", nwCfg.Name))) } return nil @@ -589,14 +589,14 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // cleanup allocated ipv4 and ipv6 addresses if they exist func (plugin *NetPlugin) cleanupAllocationOnError( - cniResults []CNIResult, + cniResults []InterfaceInfo, nwCfg *cni.NetworkConfig, args *cniSkel.CmdArgs, options map[string]interface{}, ) { - for _, cniResult := range cniResults { - if cniResult.ipResult != nil && cniResult.nicType == cns.Default { - if er := plugin.ipamInvoker.Delete(&cniResult.ipResult.IPs[0].Address, nwCfg, args, options); er != nil { + for _, interfaceInfo := range cniResults { + if interfaceInfo.ipResult != nil && interfaceInfo.nicType == cns.Infra { + if er := plugin.ipamInvoker.Delete(&interfaceInfo.ipResult.IPs[0].Address, nwCfg, args, options); er != nil { logger.Error("Failed to cleanup ip allocation on failure", zap.Error(er)) } } @@ -627,7 +627,7 @@ func (plugin *NetPlugin) createNetworkInternal( return nwInfo, err } - nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, ipamAddResult.defaultCniResult.ipResult) + nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, ipamAddResult.defaultInterfaceInfo.ipResult) if err != nil { err = plugin.Errorf("Failed to getDNSSettings: %v", err) return nwInfo, err @@ -671,7 +671,7 @@ func (plugin *NetPlugin) createNetworkInternal( // construct network info with ipv4/ipv6 subnets func addSubnetToNetworkInfo(ipamAddResult IPAMAddResult, nwInfo *network.NetworkInfo) error { - for _, ipConfig := range ipamAddResult.defaultCniResult.ipResult.IPs { + for _, ipConfig := range ipamAddResult.defaultInterfaceInfo.ipResult.IPs { ip, podSubnetPrefix, err := net.ParseCIDR(ipConfig.Address.String()) if err != nil { return fmt.Errorf("Failed to ParseCIDR for pod subnet prefix: %w", err) @@ -711,7 +711,7 @@ type createEndpointInternalOpt struct { func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) (network.EndpointInfo, error) { epInfo := network.EndpointInfo{} - defaultIPResult := opt.ipamAddResult.defaultCniResult.ipResult + defaultIPResult := opt.ipamAddResult.defaultInterfaceInfo.ipResult epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, defaultIPResult, opt.k8sNamespace) if err != nil { err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) @@ -758,8 +758,8 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) VnetCidrs: opt.nwCfg.VnetCidrs, ServiceCidrs: opt.nwCfg.ServiceCidrs, NATInfo: opt.natInfo, - NICType: cns.Default, - IsDefaultInterface: opt.ipamAddResult.defaultCniResult.isDefaultInterface, + NICType: cns.Infra, + SkipDefaultRoutes: opt.ipamAddResult.defaultInterfaceInfo.skipDefaultRoutes, } epPolicies, err := getPoliciesFromRuntimeCfg(opt.nwCfg, opt.ipamAddResult.ipv6Enabled) @@ -802,7 +802,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) epInfos := []*network.EndpointInfo{&epInfo} // get secondary interface info - for _, secondaryCniResult := range opt.ipamAddResult.cniResults { + for _, secondaryCniResult := range opt.ipamAddResult.secondaryInterfaceInfo { var addresses []net.IPNet for _, ipconfig := range secondaryCniResult.ipResult.IPs { addresses = append(addresses, ipconfig.Address) @@ -810,12 +810,12 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) epInfos = append(epInfos, &network.EndpointInfo{ - ContainerID: epInfo.ContainerID, - NetNsPath: epInfo.NetNsPath, - IPAddresses: addresses, - MacAddress: secondaryCniResult.macAddress, - NICType: cns.Secondary, - IsDefaultInterface: secondaryCniResult.isDefaultInterface, + ContainerID: epInfo.ContainerID, + NetNsPath: epInfo.NetNsPath, + IPAddresses: addresses, + MacAddress: secondaryCniResult.macAddress, + NICType: cns.Secondary, + SkipDefaultRoutes: secondaryCniResult.skipDefaultRoutes, }) } diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 3e9c0f3e22..e8b0e54984 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -161,7 +161,7 @@ func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResu // This will happen during ADD call if ipamAddResult != nil && ipamAddResult.ncResponse != nil { // networkName will look like ~ azure-vlan1-172-28-1-0_24 - ipAddrNet := ipamAddResult.defaultCniResult.ipResult.IPs[0].Address + ipAddrNet := ipamAddResult.defaultInterfaceInfo.ipResult.IPs[0].Address prefix, err := netip.ParsePrefix(ipAddrNet.String()) if err != nil { logger.Error("Error parsing network CIDR", diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index ffddfd539b..667592082f 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -182,7 +182,7 @@ func (nw *network) newEndpointImpl( return epErr } - if epInfo.NICType == cns.Default { + if epInfo.NICType == cns.Infra { var epErr error containerIf, epErr = netioCli.GetNetworkInterfaceByName(contIfName) if epErr != nil { From 88278a05aaade9717c9cf33f02d748e109853f1a Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 27 Sep 2023 19:27:41 +0000 Subject: [PATCH 54/64] update with contract changes --- cni/network/network.go | 12 ++++++------ network/endpoint.go | 6 ++++-- network/endpoint_linux.go | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index f930a9c3df..9d7e3e79ea 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -530,12 +530,12 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { if err != nil { return fmt.Errorf("IPAM Invoker Add failed with error: %w", err) } - sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam:%+v", ipamAddResult.defaultInterfaceInfo)) + sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.defaultInterfaceInfo, ipamAddResult.secondaryInterfacesInfo)) } defer func() { //nolint:gocritic if err != nil { - plugin.cleanupAllocationOnError(ipamAddResult.secondaryInterfaceInfo, nwCfg, args, options) + plugin.cleanupAllocationOnError(ipamAddResult.secondaryInterfacesInfo, nwCfg, args, options) } }() @@ -595,7 +595,7 @@ func (plugin *NetPlugin) cleanupAllocationOnError( options map[string]interface{}, ) { for _, interfaceInfo := range cniResults { - if interfaceInfo.ipResult != nil && interfaceInfo.nicType == cns.Infra { + if interfaceInfo.ipResult != nil && interfaceInfo.nicType == cns.InfraNIC { if er := plugin.ipamInvoker.Delete(&interfaceInfo.ipResult.IPs[0].Address, nwCfg, args, options); er != nil { logger.Error("Failed to cleanup ip allocation on failure", zap.Error(er)) } @@ -758,7 +758,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) VnetCidrs: opt.nwCfg.VnetCidrs, ServiceCidrs: opt.nwCfg.ServiceCidrs, NATInfo: opt.natInfo, - NICType: cns.Infra, + NICType: cns.InfraNIC, SkipDefaultRoutes: opt.ipamAddResult.defaultInterfaceInfo.skipDefaultRoutes, } @@ -802,7 +802,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) epInfos := []*network.EndpointInfo{&epInfo} // get secondary interface info - for _, secondaryCniResult := range opt.ipamAddResult.secondaryInterfaceInfo { + for _, secondaryCniResult := range opt.ipamAddResult.secondaryInterfacesInfo { var addresses []net.IPNet for _, ipconfig := range secondaryCniResult.ipResult.IPs { addresses = append(addresses, ipconfig.Address) @@ -814,7 +814,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) NetNsPath: epInfo.NetNsPath, IPAddresses: addresses, MacAddress: secondaryCniResult.macAddress, - NICType: cns.Secondary, + NICType: cns.DelegatedVMNIC, SkipDefaultRoutes: secondaryCniResult.skipDefaultRoutes, }) } diff --git a/network/endpoint.go b/network/endpoint.go index 4b45f59028..78935a3b5c 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/Azure/azure-container-networking/cni/log" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/network/policy" @@ -88,7 +89,7 @@ type EndpointInfo struct { VnetCidrs string ServiceCidrs string NATInfo []policy.NATInfo - NICType string + NICType cns.NICType SkipDefaultRoutes bool } @@ -104,12 +105,13 @@ type RouteInfo struct { Table int } +// InterfaceInfo contains information for secondary interfaces type InterfaceInfo struct { Name string MacAddress net.HardwareAddr IPAddress []net.IPNet Routes []RouteInfo - NICType string + NICType cns.NICType SkipDefaultRoutes bool } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 667592082f..b17e93d47a 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -153,7 +153,7 @@ func (nw *network) newEndpointImpl( } else if nw.Mode != opModeTransparent { logger.Info("Bridge client") epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) - } else if epInfo.NICType == cns.Secondary { + } else if epInfo.NICType == cns.DelegatedVMNIC { logger.Info("Secondary client") epClient = NewSecondaryEndpointClient(nl, plc, ep) } else { @@ -182,7 +182,7 @@ func (nw *network) newEndpointImpl( return epErr } - if epInfo.NICType == cns.Infra { + if epInfo.NICType == cns.InfraNIC { var epErr error containerIf, epErr = netioCli.GetNetworkInterfaceByName(contIfName) if epErr != nil { From 091b8e426565165832686a880ef54b6d4a5f5174 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 4 Oct 2023 22:52:33 +0000 Subject: [PATCH 55/64] add more tests and address comments --- cni/network/network.go | 15 +++++++---- cni/network/network_test.go | 52 +++++++++++++++++++++++++++---------- network/manager.go | 2 +- network/manager_mock.go | 12 ++++++++- 4 files changed, 61 insertions(+), 20 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 9d7e3e79ea..cbd9a66e25 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -535,7 +535,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { defer func() { //nolint:gocritic if err != nil { - plugin.cleanupAllocationOnError(ipamAddResult.secondaryInterfacesInfo, nwCfg, args, options) + plugin.cleanupAllocationOnError(ipamAddResult.defaultInterfaceInfo.ipResult, nwCfg, args, options) } }() @@ -589,14 +589,14 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // cleanup allocated ipv4 and ipv6 addresses if they exist func (plugin *NetPlugin) cleanupAllocationOnError( - cniResults []InterfaceInfo, + result *cniTypesCurr.Result, nwCfg *cni.NetworkConfig, args *cniSkel.CmdArgs, options map[string]interface{}, ) { - for _, interfaceInfo := range cniResults { - if interfaceInfo.ipResult != nil && interfaceInfo.nicType == cns.InfraNIC { - if er := plugin.ipamInvoker.Delete(&interfaceInfo.ipResult.IPs[0].Address, nwCfg, args, options); er != nil { + if result != nil { + for i := 0; i < len(result.IPs); i++ { + if er := plugin.ipamInvoker.Delete(&result.IPs[i].Address, nwCfg, args, options); er != nil { logger.Error("Failed to cleanup ip allocation on failure", zap.Error(er)) } } @@ -804,15 +804,20 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) // get secondary interface info for _, secondaryCniResult := range opt.ipamAddResult.secondaryInterfacesInfo { var addresses []net.IPNet + var routes []network.RouteInfo for _, ipconfig := range secondaryCniResult.ipResult.IPs { addresses = append(addresses, ipconfig.Address) } + for _, route := range secondaryCniResult.ipResult.Routes { + routes = append(routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) + } epInfos = append(epInfos, &network.EndpointInfo{ ContainerID: epInfo.ContainerID, NetNsPath: epInfo.NetNsPath, IPAddresses: addresses, + Routes: routes, MacAddress: secondaryCniResult.macAddress, NICType: cns.DelegatedVMNIC, SkipDefaultRoutes: secondaryCniResult.skipDefaultRoutes, diff --git a/cni/network/network_test.go b/cni/network/network_test.go index c75a79c830..ed963a6241 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -75,7 +75,7 @@ func GetTestResources() *NetPlugin { grpcClient := &nns.MockGrpcClient{} plugin, _ := NewPlugin(pluginName, config, grpcClient, &Multitenancy{}) plugin.report = &telemetry.CNIReport{} - mockNetworkManager := acnnetwork.NewMockNetworkmanager() + mockNetworkManager := acnnetwork.NewMockNetworkmanager(false) plugin.nm = mockNetworkManager plugin.ipamInvoker = NewMockIpamInvoker(isIPv6, false, false, false, false) return plugin @@ -310,6 +310,7 @@ func TestIpamAddFail(t *testing.T) { methods []string cniArgs []cniSkel.CmdArgs wantErr []bool + wantEndpointErr bool wantErrMsg string expectedEndpoints int }{ @@ -366,6 +367,23 @@ func TestIpamAddFail(t *testing.T) { wantErrMsg: "v4 fail", expectedEndpoints: 1, }, + { + name: "cleanup ipam add fail", + methods: []string{CNI_ADD}, + cniArgs: []cniSkel.CmdArgs{ + { + ContainerID: "test1-container", + Netns: "test1-container", + StdinData: nwCfg.Serialize(), + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "container1", "container1-ns"), + IfName: eth0IfName, + }, + }, + wantErr: []bool{false}, + wantEndpointErr: true, + wantErrMsg: "Failed to create endpoint: create endpoint failed", + expectedEndpoints: 0, + }, } for _, tt := range tests { @@ -380,18 +398,26 @@ func TestIpamAddFail(t *testing.T) { plugin.ipamInvoker = NewMockIpamInvoker(false, false, false, false, false) } + if tt.wantEndpointErr { + plugin.nm = acnnetwork.NewMockNetworkmanager(true) + } + if method == CNI_ADD { err = plugin.Add(&tt.cniArgs[i]) } else if method == CNI_DEL { err = plugin.Delete(&tt.cniArgs[i]) } - if tt.wantErr[i] { + if tt.wantErr[i] || tt.wantEndpointErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.wantErrMsg) } else { require.NoError(t, err) } + + if tt.wantEndpointErr { + assert.Len(t, plugin.ipamInvoker.(*MockIpamInvoker).ipMap, 0) + } } }) @@ -459,7 +485,7 @@ func TestAddDualStack(t *testing.T) { name: "Dualstack happy path", plugin: &NetPlugin{ Plugin: cniPlugin, - nm: acnnetwork.NewMockNetworkmanager(), + nm: acnnetwork.NewMockNetworkmanager(false), ipamInvoker: NewMockIpamInvoker(true, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -470,7 +496,7 @@ func TestAddDualStack(t *testing.T) { name: "Dualstack ipv6 fail", plugin: &NetPlugin{ Plugin: cniPlugin, - nm: acnnetwork.NewMockNetworkmanager(), + nm: acnnetwork.NewMockNetworkmanager(false), ipamInvoker: NewMockIpamInvoker(true, false, true, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -516,7 +542,7 @@ func TestPluginGet(t *testing.T) { methods: []string{CNI_ADD, "GET"}, plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(), + nm: acnnetwork.NewMockNetworkmanager(false), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -528,7 +554,7 @@ func TestPluginGet(t *testing.T) { methods: []string{"GET"}, plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(), + nm: acnnetwork.NewMockNetworkmanager(false), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -541,7 +567,7 @@ func TestPluginGet(t *testing.T) { methods: []string{CNI_ADD, CNI_DEL, "GET"}, plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(), + nm: acnnetwork.NewMockNetworkmanager(false), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -604,7 +630,7 @@ func TestPluginMultitenancyAdd(t *testing.T) { name: "Add Happy path", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(), + nm: acnnetwork.NewMockNetworkmanager(false), tb: &telemetry.TelemetryBuffer{}, report: &telemetry.CNIReport{}, multitenancyClient: NewMockMultitenancy(false), @@ -623,7 +649,7 @@ func TestPluginMultitenancyAdd(t *testing.T) { name: "Add Fail", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(), + nm: acnnetwork.NewMockNetworkmanager(false), tb: &telemetry.TelemetryBuffer{}, report: &telemetry.CNIReport{}, multitenancyClient: NewMockMultitenancy(true), @@ -735,7 +761,7 @@ func TestPluginBaremetalAdd(t *testing.T) { name: "Baremetal Add Happy path", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(), + nm: acnnetwork.NewMockNetworkmanager(false), tb: &telemetry.TelemetryBuffer{}, report: &telemetry.CNIReport{}, nnsClient: &nns.MockGrpcClient{}, @@ -753,7 +779,7 @@ func TestPluginBaremetalAdd(t *testing.T) { name: "Baremetal Add Fail", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(), + nm: acnnetwork.NewMockNetworkmanager(false), tb: &telemetry.TelemetryBuffer{}, report: &telemetry.CNIReport{}, nnsClient: &nns.MockGrpcClient{Fail: true}, @@ -1126,7 +1152,7 @@ func TestPluginSwiftV2Add(t *testing.T) { name: "SwiftV2 Add Happy path", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(), + nm: acnnetwork.NewMockNetworkmanager(false), ipamInvoker: NewMockIpamInvoker(false, false, false, true, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -1144,7 +1170,7 @@ func TestPluginSwiftV2Add(t *testing.T) { name: "SwiftV2 Add fail", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(), + nm: acnnetwork.NewMockNetworkmanager(false), ipamInvoker: NewMockIpamInvoker(false, false, false, true, true), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, diff --git a/network/manager.go b/network/manager.go index 2bda9f23a2..99e3b082bd 100644 --- a/network/manager.go +++ b/network/manager.go @@ -337,7 +337,7 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn } if nw.VlanId != 0 { - // the first entry in epInfo is default address type + // the first entry in epInfo is InfraNIC type if epInfo[0].Data[VlanIDKey] == nil { logger.Info("overriding endpoint vlanid with network vlanid") epInfo[0].Data[VlanIDKey] = nw.VlanId diff --git a/network/manager_mock.go b/network/manager_mock.go index a77623e5d6..70768dbd8c 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -1,21 +1,27 @@ package network import ( + "errors" + cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" "github.com/Azure/azure-container-networking/common" ) +var errCreateEndpointFailed = errors.New("create endpoint failed") + // MockNetworkManager is a mock structure for Network Manager type MockNetworkManager struct { TestNetworkInfoMap map[string]*NetworkInfo TestEndpointInfoMap map[string]*EndpointInfo + createEndpointFail bool } // NewMockNetworkmanager returns a new mock -func NewMockNetworkmanager() *MockNetworkManager { +func NewMockNetworkmanager(createEndpointFail bool) *MockNetworkManager { return &MockNetworkManager{ TestNetworkInfoMap: make(map[string]*NetworkInfo), TestEndpointInfoMap: make(map[string]*EndpointInfo), + createEndpointFail: createEndpointFail, } } @@ -53,6 +59,10 @@ func (nm *MockNetworkManager) GetNetworkInfo(networkID string) (NetworkInfo, err // CreateEndpoint mock func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfo []*EndpointInfo) error { + if nm.createEndpointFail { + return errCreateEndpointFailed + } + nm.TestEndpointInfoMap[epInfo[0].Id] = epInfo[0] return nil } From 10ba52f0dc0317908cdc2bf0aa9d1be12b151aa9 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Thu, 5 Oct 2023 16:24:55 +0000 Subject: [PATCH 56/64] modify AddEndpoints per comments --- network/endpoint_linux.go | 14 ++------------ network/endpoint_test.go | 2 +- network/manager.go | 2 +- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index b17e93d47a..54a7855fa6 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -58,7 +58,6 @@ func (nw *network) newEndpointImpl( epInfo []*EndpointInfo, ) (*endpoint, error) { var ( - containerIf *net.Interface err error hostIfName string contIfName string @@ -167,7 +166,7 @@ func (nw *network) newEndpointImpl( // Cleanup on failure. if err != nil { logger.Error("CNI error. Delete Endpoint and rules that are created", zap.Error(err), zap.String("contIfName", contIfName)) - if containerIf != nil { + if ep.MacAddress != nil { client.DeleteEndpointRules(ep) } // set deleteHostVeth to true to cleanup host veth interface if created @@ -178,19 +177,10 @@ func (nw *network) newEndpointImpl( //nolint:wrapcheck // ignore wrap check err = func() error { - if epErr := epClient.AddEndpoints(epInfo); epErr != nil { + if epErr := epClient.AddEndpoints(epInfo, ep); epErr != nil { return epErr } - if epInfo.NICType == cns.InfraNIC { - var epErr error - containerIf, epErr = netioCli.GetNetworkInterfaceByName(contIfName) - if epErr != nil { - return epErr - } - ep.MacAddress = containerIf.HardwareAddr - } - // Setup rules for IP addresses on the container interface. if epErr := epClient.AddEndpointRules(epInfo); epErr != nil { return epErr diff --git a/network/endpoint_test.go b/network/endpoint_test.go index 16a0e71d0f..f43f4dfe73 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -205,7 +205,7 @@ var _ = Describe("Test Endpoint", func() { It("Should be not added", func() { // Adding an endpoint with an id. mockCli := NewMockEndpointClient(false) - err := mockCli.AddEndpoints(epInfo) + err := mockCli.AddEndpoints(epInfo, nil) Expect(err).ToNot(HaveOccurred()) // Adding endpoint with same id should fail and delete should cleanup the state ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), diff --git a/network/manager.go b/network/manager.go index 99e3b082bd..25512c28b2 100644 --- a/network/manager.go +++ b/network/manager.go @@ -50,7 +50,7 @@ type NetworkClient interface { } type EndpointClient interface { - AddEndpoints(epInfo *EndpointInfo) error + AddEndpoints(epInfo *EndpointInfo, ep *endpoint) error AddEndpointRules(epInfo *EndpointInfo) error DeleteEndpointRules(ep *endpoint) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error From ff5f7a7470022385caecbf4f6633d2be42babcab Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 9 Oct 2023 17:18:30 +0000 Subject: [PATCH 57/64] update test for invoker add and endpoint client add failure --- cni/network/network_test.go | 50 ++++++++++++++++++++++++---------- network/endpoint_test.go | 11 +++++--- network/manager_mock.go | 20 ++++++-------- network/mock_endpointclient.go | 10 +++++++ 4 files changed, 61 insertions(+), 30 deletions(-) diff --git a/cni/network/network_test.go b/cni/network/network_test.go index ed963a6241..de4cdf1749 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -75,7 +75,7 @@ func GetTestResources() *NetPlugin { grpcClient := &nns.MockGrpcClient{} plugin, _ := NewPlugin(pluginName, config, grpcClient, &Multitenancy{}) plugin.report = &telemetry.CNIReport{} - mockNetworkManager := acnnetwork.NewMockNetworkmanager(false) + mockNetworkManager := acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)) plugin.nm = mockNetworkManager plugin.ipamInvoker = NewMockIpamInvoker(isIPv6, false, false, false, false) return plugin @@ -381,7 +381,7 @@ func TestIpamAddFail(t *testing.T) { }, wantErr: []bool{false}, wantEndpointErr: true, - wantErrMsg: "Failed to create endpoint: create endpoint failed", + wantErrMsg: "Failed to create endpoint: MockEndpointClient Error : AddEndpoints failed", expectedEndpoints: 0, }, } @@ -399,7 +399,7 @@ func TestIpamAddFail(t *testing.T) { } if tt.wantEndpointErr { - plugin.nm = acnnetwork.NewMockNetworkmanager(true) + plugin.nm = acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(true)) } if method == CNI_ADD { @@ -485,7 +485,7 @@ func TestAddDualStack(t *testing.T) { name: "Dualstack happy path", plugin: &NetPlugin{ Plugin: cniPlugin, - nm: acnnetwork.NewMockNetworkmanager(false), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), ipamInvoker: NewMockIpamInvoker(true, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -496,7 +496,7 @@ func TestAddDualStack(t *testing.T) { name: "Dualstack ipv6 fail", plugin: &NetPlugin{ Plugin: cniPlugin, - nm: acnnetwork.NewMockNetworkmanager(false), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), ipamInvoker: NewMockIpamInvoker(true, false, true, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -542,7 +542,7 @@ func TestPluginGet(t *testing.T) { methods: []string{CNI_ADD, "GET"}, plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(false), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -554,7 +554,7 @@ func TestPluginGet(t *testing.T) { methods: []string{"GET"}, plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(false), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -567,7 +567,7 @@ func TestPluginGet(t *testing.T) { methods: []string{CNI_ADD, CNI_DEL, "GET"}, plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(false), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -630,7 +630,7 @@ func TestPluginMultitenancyAdd(t *testing.T) { name: "Add Happy path", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(false), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), tb: &telemetry.TelemetryBuffer{}, report: &telemetry.CNIReport{}, multitenancyClient: NewMockMultitenancy(false), @@ -649,7 +649,7 @@ func TestPluginMultitenancyAdd(t *testing.T) { name: "Add Fail", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(false), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), tb: &telemetry.TelemetryBuffer{}, report: &telemetry.CNIReport{}, multitenancyClient: NewMockMultitenancy(true), @@ -761,7 +761,7 @@ func TestPluginBaremetalAdd(t *testing.T) { name: "Baremetal Add Happy path", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(false), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), tb: &telemetry.TelemetryBuffer{}, report: &telemetry.CNIReport{}, nnsClient: &nns.MockGrpcClient{}, @@ -779,7 +779,7 @@ func TestPluginBaremetalAdd(t *testing.T) { name: "Baremetal Add Fail", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(false), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), tb: &telemetry.TelemetryBuffer{}, report: &telemetry.CNIReport{}, nnsClient: &nns.MockGrpcClient{Fail: true}, @@ -1152,7 +1152,7 @@ func TestPluginSwiftV2Add(t *testing.T) { name: "SwiftV2 Add Happy path", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(false), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), ipamInvoker: NewMockIpamInvoker(false, false, false, true, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -1167,10 +1167,10 @@ func TestPluginSwiftV2Add(t *testing.T) { wantErr: false, }, { - name: "SwiftV2 Add fail", + name: "SwiftV2 Invoker Add fail", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(false), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), ipamInvoker: NewMockIpamInvoker(false, false, false, true, true), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -1184,6 +1184,24 @@ func TestPluginSwiftV2Add(t *testing.T) { }, wantErr: true, }, + { + name: "SwiftV2 EndpointClient Add fail", + plugin: &NetPlugin{ + Plugin: plugin, + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(true)), + ipamInvoker: NewMockIpamInvoker(false, false, false, true, false), + report: &telemetry.CNIReport{}, + tb: &telemetry.TelemetryBuffer{}, + }, + args: &cniSkel.CmdArgs{ + StdinData: localNwCfg.Serialize(), + ContainerID: "test-container", + Netns: "test-container", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + }, + wantErr: true, + }, } for _, tt := range tests { @@ -1193,6 +1211,8 @@ func TestPluginSwiftV2Add(t *testing.T) { if tt.wantErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.wantErrMsg, "Expected %v but got %+v", tt.wantErrMsg, err.Error()) + endpoints, _ := tt.plugin.nm.GetAllEndpoints(localNwCfg.Name) + require.Condition(t, assert.Comparison(func() bool { return len(endpoints) == 0 })) } else { require.NoError(t, err) endpoints, _ := tt.plugin.nm.GetAllEndpoints(localNwCfg.Name) diff --git a/network/endpoint_test.go b/network/endpoint_test.go index f43f4dfe73..ebf35d6466 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -173,8 +173,9 @@ var _ = Describe("Test Endpoint", func() { Endpoints: map[string]*endpoint{}, } epInfo := &EndpointInfo{ - Id: "768e8deb-eth1", - Data: make(map[string]interface{}), + Id: "768e8deb-eth1", + Data: make(map[string]interface{}), + IfName: eth0IfName, } epInfo.Data[VlanIDKey] = 100 @@ -239,7 +240,8 @@ var _ = Describe("Test Endpoint", func() { Endpoints: map[string]*endpoint{}, } epInfo := &EndpointInfo{ - Id: "768e8deb-eth1", + Id: "768e8deb-eth1", + IfName: eth0IfName, } ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), netio.NewMockNetIO(false, 0), NewMockEndpointClient(true), []*EndpointInfo{epInfo}) @@ -261,7 +263,8 @@ var _ = Describe("Test Endpoint", func() { nw := &network{} existingEpInfo := &EndpointInfo{ - Id: "768e8deb-eth1", + Id: "768e8deb-eth1", + IfName: eth0IfName, } targetEpInfo := &EndpointInfo{} err := nm.updateEndpoint(nw, existingEpInfo, targetEpInfo) diff --git a/network/manager_mock.go b/network/manager_mock.go index 70768dbd8c..f5552e8401 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -1,27 +1,23 @@ package network import ( - "errors" - cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" "github.com/Azure/azure-container-networking/common" ) -var errCreateEndpointFailed = errors.New("create endpoint failed") - // MockNetworkManager is a mock structure for Network Manager type MockNetworkManager struct { TestNetworkInfoMap map[string]*NetworkInfo TestEndpointInfoMap map[string]*EndpointInfo - createEndpointFail bool + TestEndpointClient *MockEndpointClient } // NewMockNetworkmanager returns a new mock -func NewMockNetworkmanager(createEndpointFail bool) *MockNetworkManager { +func NewMockNetworkmanager(mockEndpointclient *MockEndpointClient) *MockNetworkManager { return &MockNetworkManager{ TestNetworkInfoMap: make(map[string]*NetworkInfo), TestEndpointInfoMap: make(map[string]*EndpointInfo), - createEndpointFail: createEndpointFail, + TestEndpointClient: mockEndpointclient, } } @@ -58,12 +54,14 @@ func (nm *MockNetworkManager) GetNetworkInfo(networkID string) (NetworkInfo, err } // CreateEndpoint mock -func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfo []*EndpointInfo) error { - if nm.createEndpointFail { - return errCreateEndpointFailed +func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfos []*EndpointInfo) error { + for _, epInfo := range epInfos { + if err := nm.TestEndpointClient.AddEndpoints(epInfo, nil); err != nil { + return err + } } - nm.TestEndpointInfoMap[epInfo[0].Id] = epInfo[0] + nm.TestEndpointInfoMap[epInfos[0].Id] = epInfos[0] return nil } diff --git a/network/mock_endpointclient.go b/network/mock_endpointclient.go index b95ab47890..f2f0e9a86d 100644 --- a/network/mock_endpointclient.go +++ b/network/mock_endpointclient.go @@ -11,7 +11,11 @@ const ( eth0IfName = "eth0" ) +<<<<<<< HEAD func NewErrorMockEndpointClient(errStr string) error { +======= +func newErrorMockEndpointClient(errStr string) error { +>>>>>>> b657c80b (update test for invoker add and endpoint client add failure) return fmt.Errorf("%w : %s", errMockEpClient, errStr) } @@ -35,9 +39,15 @@ func NewMockEndpointClient(fn func(*EndpointInfo) error) *MockEndpointClient { return client } +<<<<<<< HEAD func (client *MockEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if ok := client.endpoints[epInfo.Id]; ok && epInfo.IfName == eth0IfName { return NewErrorMockEndpointClient("Endpoint already exists") +======= +func (client *MockEndpointClient) AddEndpoints(epInfo *EndpointInfo, _ *endpoint) error { + if ok := client.endpoints[epInfo.Id]; ok && epInfo.IfName == eth0IfName { + return newErrorMockEndpointClient("Endpoint already exists") +>>>>>>> b657c80b (update test for invoker add and endpoint client add failure) } client.endpoints[epInfo.Id] = true From 13bd2e1411a02717d0dc6c846748f2cf22f7812e Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 9 Oct 2023 22:00:36 +0000 Subject: [PATCH 58/64] address comments --- cni/network/network_test.go | 49 +++++++++++++++++++++------------- network/endpoint_linux.go | 16 +++++++++-- network/endpoint_test.go | 26 +++++++++++------- network/manager.go | 2 +- network/manager_mock.go | 2 +- network/mock_endpointclient.go | 10 ------- 6 files changed, 63 insertions(+), 42 deletions(-) diff --git a/cni/network/network_test.go b/cni/network/network_test.go index de4cdf1749..0df2010a8b 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -10,6 +10,7 @@ import ( "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cni/api" "github.com/Azure/azure-container-networking/cni/util" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/common" acnnetwork "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/network/networkutils" @@ -75,7 +76,7 @@ func GetTestResources() *NetPlugin { grpcClient := &nns.MockGrpcClient{} plugin, _ := NewPlugin(pluginName, config, grpcClient, &Multitenancy{}) plugin.report = &telemetry.CNIReport{} - mockNetworkManager := acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)) + mockNetworkManager := acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)) plugin.nm = mockNetworkManager plugin.ipamInvoker = NewMockIpamInvoker(isIPv6, false, false, false, false) return plugin @@ -381,7 +382,7 @@ func TestIpamAddFail(t *testing.T) { }, wantErr: []bool{false}, wantEndpointErr: true, - wantErrMsg: "Failed to create endpoint: MockEndpointClient Error : AddEndpoints failed", + wantErrMsg: "Failed to create endpoint: MockEndpointClient Error : Endpoint Error", expectedEndpoints: 0, }, } @@ -399,7 +400,9 @@ func TestIpamAddFail(t *testing.T) { } if tt.wantEndpointErr { - plugin.nm = acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(true)) + plugin.nm = acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(func(*acnnetwork.EndpointInfo) error { + return acnnetwork.NewErrorMockEndpointClient("Endpoint Error") //nolint:wrapcheck // ignore wrapping for test + })) } if method == CNI_ADD { @@ -485,7 +488,7 @@ func TestAddDualStack(t *testing.T) { name: "Dualstack happy path", plugin: &NetPlugin{ Plugin: cniPlugin, - nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(true, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -496,7 +499,7 @@ func TestAddDualStack(t *testing.T) { name: "Dualstack ipv6 fail", plugin: &NetPlugin{ Plugin: cniPlugin, - nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(true, false, true, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -542,7 +545,7 @@ func TestPluginGet(t *testing.T) { methods: []string{CNI_ADD, "GET"}, plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -554,7 +557,7 @@ func TestPluginGet(t *testing.T) { methods: []string{"GET"}, plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -567,7 +570,7 @@ func TestPluginGet(t *testing.T) { methods: []string{CNI_ADD, CNI_DEL, "GET"}, plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(false, false, false, false, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -630,7 +633,7 @@ func TestPluginMultitenancyAdd(t *testing.T) { name: "Add Happy path", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), tb: &telemetry.TelemetryBuffer{}, report: &telemetry.CNIReport{}, multitenancyClient: NewMockMultitenancy(false), @@ -649,7 +652,7 @@ func TestPluginMultitenancyAdd(t *testing.T) { name: "Add Fail", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), tb: &telemetry.TelemetryBuffer{}, report: &telemetry.CNIReport{}, multitenancyClient: NewMockMultitenancy(true), @@ -761,7 +764,7 @@ func TestPluginBaremetalAdd(t *testing.T) { name: "Baremetal Add Happy path", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), tb: &telemetry.TelemetryBuffer{}, report: &telemetry.CNIReport{}, nnsClient: &nns.MockGrpcClient{}, @@ -779,7 +782,7 @@ func TestPluginBaremetalAdd(t *testing.T) { name: "Baremetal Add Fail", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), tb: &telemetry.TelemetryBuffer{}, report: &telemetry.CNIReport{}, nnsClient: &nns.MockGrpcClient{Fail: true}, @@ -1152,7 +1155,7 @@ func TestPluginSwiftV2Add(t *testing.T) { name: "SwiftV2 Add Happy path", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(false, false, false, true, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -1170,7 +1173,7 @@ func TestPluginSwiftV2Add(t *testing.T) { name: "SwiftV2 Invoker Add fail", plugin: &NetPlugin{ Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(false)), + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(nil)), ipamInvoker: NewMockIpamInvoker(false, false, false, true, true), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -1182,13 +1185,20 @@ func TestPluginSwiftV2Add(t *testing.T) { Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), IfName: eth0IfName, }, - wantErr: true, + wantErr: true, + wantErrMsg: "IPAM Invoker Add failed with error: delegatedVMNIC fail", }, { name: "SwiftV2 EndpointClient Add fail", plugin: &NetPlugin{ - Plugin: plugin, - nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(true)), + Plugin: plugin, + nm: acnnetwork.NewMockNetworkmanager(acnnetwork.NewMockEndpointClient(func(ep *acnnetwork.EndpointInfo) error { + if ep.NICType == cns.DelegatedVMNIC { + return acnnetwork.NewErrorMockEndpointClient("AddEndpoints Delegated VM NIC failed") //nolint:wrapcheck // ignore wrapping for test + } + + return nil + })), ipamInvoker: NewMockIpamInvoker(false, false, false, true, false), report: &telemetry.CNIReport{}, tb: &telemetry.TelemetryBuffer{}, @@ -1200,7 +1210,8 @@ func TestPluginSwiftV2Add(t *testing.T) { Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), IfName: eth0IfName, }, - wantErr: true, + wantErr: true, + wantErrMsg: "Failed to create endpoint: MockEndpointClient Error : AddEndpoints Delegated VM NIC failed", }, } @@ -1210,7 +1221,7 @@ func TestPluginSwiftV2Add(t *testing.T) { err := tt.plugin.Add(tt.args) if tt.wantErr { require.Error(t, err) - assert.Contains(t, err.Error(), tt.wantErrMsg, "Expected %v but got %+v", tt.wantErrMsg, err.Error()) + assert.Equal(t, err.Error(), tt.wantErrMsg, "Expected %v but got %+v", tt.wantErrMsg, err.Error()) endpoints, _ := tt.plugin.nm.GetAllEndpoints(localNwCfg.Name) require.Condition(t, assert.Comparison(func() bool { return len(endpoints) == 0 })) } else { diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 54a7855fa6..eb74838d39 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -64,6 +64,7 @@ func (nw *network) newEndpointImpl( localIP string vlanid = 0 defaultEpInfo = epInfo[0] + containerIf *net.Interface ) if nw.Endpoints[defaultEpInfo.Id] != nil { @@ -122,6 +123,7 @@ func (nw *network) newEndpointImpl( for _, epInfo := range epInfo { // testEpClient is non-nil only when the endpoint is created for the unit test + // resetting epClient to testEpClient in loop to use the test endpoint client if specified epClient := testEpClient if epClient == nil { //nolint:gocritic @@ -166,7 +168,7 @@ func (nw *network) newEndpointImpl( // Cleanup on failure. if err != nil { logger.Error("CNI error. Delete Endpoint and rules that are created", zap.Error(err), zap.String("contIfName", contIfName)) - if ep.MacAddress != nil { + if containerIf != nil { client.DeleteEndpointRules(ep) } // set deleteHostVeth to true to cleanup host veth interface if created @@ -175,12 +177,22 @@ func (nw *network) newEndpointImpl( } }(epClient, contIfName) + // wrapping endpoint client commands in anonymous func so that namespace can be exit and closed before the next loop //nolint:wrapcheck // ignore wrap check err = func() error { - if epErr := epClient.AddEndpoints(epInfo, ep); epErr != nil { + if epErr := epClient.AddEndpoints(epInfo); epErr != nil { return epErr } + if epInfo.NICType == cns.InfraNIC { + var epErr error + containerIf, epErr = netioCli.GetNetworkInterfaceByName(contIfName) + if epErr != nil { + return epErr + } + ep.MacAddress = containerIf.HardwareAddr + } + // Setup rules for IP addresses on the container interface. if epErr := epClient.AddEndpointRules(epInfo); epErr != nil { return epErr diff --git a/network/endpoint_test.go b/network/endpoint_test.go index ebf35d6466..803e08adaf 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -7,6 +7,7 @@ import ( "net" "testing" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/netio" "github.com/Azure/azure-container-networking/netlink" "github.com/Azure/azure-container-networking/platform" @@ -182,7 +183,7 @@ var _ = Describe("Test Endpoint", func() { It("Should be added", func() { // Add endpoint with valid id ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(false), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -194,7 +195,7 @@ var _ = Describe("Test Endpoint", func() { extIf: &externalInterface{IPv4Gateway: net.ParseIP("192.168.0.1")}, } ep, err := nw2.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(false), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -205,8 +206,8 @@ var _ = Describe("Test Endpoint", func() { }) It("Should be not added", func() { // Adding an endpoint with an id. - mockCli := NewMockEndpointClient(false) - err := mockCli.AddEndpoints(epInfo, nil) + mockCli := NewMockEndpointClient(nil) + err := mockCli.AddEndpoints(epInfo) Expect(err).ToNot(HaveOccurred()) // Adding endpoint with same id should fail and delete should cleanup the state ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), @@ -218,7 +219,7 @@ var _ = Describe("Test Endpoint", func() { }) It("Should be deleted", func() { // Adding an endpoint with an id. - mockCli := NewMockEndpointClient(false) + mockCli := NewMockEndpointClient(nil) ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), netio.NewMockNetIO(false, 0), mockCli, []*EndpointInfo{epInfo}) Expect(err).ToNot(HaveOccurred()) @@ -240,15 +241,22 @@ var _ = Describe("Test Endpoint", func() { Endpoints: map[string]*endpoint{}, } epInfo := &EndpointInfo{ - Id: "768e8deb-eth1", - IfName: eth0IfName, + Id: "768e8deb-eth1", + IfName: eth0IfName, + NICType: cns.InfraNIC, } ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(true), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(func(ep *EndpointInfo) error { + if ep.NICType == cns.InfraNIC { + return NewErrorMockEndpointClient("AddEndpoints Infra NIC failed") + } + + return nil + }), []*EndpointInfo{epInfo}) Expect(err).To(HaveOccurred()) Expect(ep).To(BeNil()) ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(false), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) diff --git a/network/manager.go b/network/manager.go index 25512c28b2..99e3b082bd 100644 --- a/network/manager.go +++ b/network/manager.go @@ -50,7 +50,7 @@ type NetworkClient interface { } type EndpointClient interface { - AddEndpoints(epInfo *EndpointInfo, ep *endpoint) error + AddEndpoints(epInfo *EndpointInfo) error AddEndpointRules(epInfo *EndpointInfo) error DeleteEndpointRules(ep *endpoint) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error diff --git a/network/manager_mock.go b/network/manager_mock.go index f5552e8401..db5cd024f4 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -56,7 +56,7 @@ func (nm *MockNetworkManager) GetNetworkInfo(networkID string) (NetworkInfo, err // CreateEndpoint mock func (nm *MockNetworkManager) CreateEndpoint(_ apipaClient, _ string, epInfos []*EndpointInfo) error { for _, epInfo := range epInfos { - if err := nm.TestEndpointClient.AddEndpoints(epInfo, nil); err != nil { + if err := nm.TestEndpointClient.AddEndpoints(epInfo); err != nil { return err } } diff --git a/network/mock_endpointclient.go b/network/mock_endpointclient.go index f2f0e9a86d..b95ab47890 100644 --- a/network/mock_endpointclient.go +++ b/network/mock_endpointclient.go @@ -11,11 +11,7 @@ const ( eth0IfName = "eth0" ) -<<<<<<< HEAD func NewErrorMockEndpointClient(errStr string) error { -======= -func newErrorMockEndpointClient(errStr string) error { ->>>>>>> b657c80b (update test for invoker add and endpoint client add failure) return fmt.Errorf("%w : %s", errMockEpClient, errStr) } @@ -39,15 +35,9 @@ func NewMockEndpointClient(fn func(*EndpointInfo) error) *MockEndpointClient { return client } -<<<<<<< HEAD func (client *MockEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { if ok := client.endpoints[epInfo.Id]; ok && epInfo.IfName == eth0IfName { return NewErrorMockEndpointClient("Endpoint already exists") -======= -func (client *MockEndpointClient) AddEndpoints(epInfo *EndpointInfo, _ *endpoint) error { - if ok := client.endpoints[epInfo.Id]; ok && epInfo.IfName == eth0IfName { - return newErrorMockEndpointClient("Endpoint already exists") ->>>>>>> b657c80b (update test for invoker add and endpoint client add failure) } client.endpoints[epInfo.Id] = true From bbdb5ea09aae0ef1c47b61d134d11e66168e2b6c Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 11 Oct 2023 15:37:46 +0000 Subject: [PATCH 59/64] fix lint --- network/endpoint_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index eb74838d39..7d45c3d61d 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -222,7 +222,7 @@ func (nw *network) newEndpointImpl( defer func() { logger.Info("Exiting netns", zap.Any("NetNsPath", epInfo.NetNsPath)) if epErr := ns.Exit(); epErr != nil { - logger.Error("Failed to exit netns with", zap.Error(err)) + logger.Error("Failed to exit netns with", zap.Error(epErr)) } }() } From 9db3feb817d4320c9a9a0aa77fde1def4b0031a6 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 11 Oct 2023 16:31:19 +0000 Subject: [PATCH 60/64] update windows tests --- cni/network/network_windows_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index 04180ae0b1..48cb9f0c53 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -367,7 +367,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ID: 1, }, }, - defaultCniResult: CNIResult{ + defaultInterfaceInfo: InterfaceInfo{ ipResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { @@ -404,7 +404,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ID: 1, }, }, - defaultCniResult: CNIResult{ + defaultInterfaceInfo: InterfaceInfo{ ipResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { @@ -441,7 +441,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ID: 1, }, }, - defaultCniResult: CNIResult{ + defaultInterfaceInfo: InterfaceInfo{ ipResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { @@ -478,7 +478,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { ID: 1, }, }, - defaultCniResult: CNIResult{ + defaultInterfaceInfo: InterfaceInfo{ ipResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { @@ -511,7 +511,7 @@ func TestGetNetworkNameFromCNS(t *testing.T) { }, ipamAddResult: &IPAMAddResult{ ncResponse: &cns.GetNetworkContainerResponse{}, - defaultCniResult: CNIResult{ + defaultInterfaceInfo: InterfaceInfo{ ipResult: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { From f2563b324c341f92d5446f7cc4ec01feefd7a67e Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 18 Oct 2023 15:11:36 +0000 Subject: [PATCH 61/64] update refactor with namespace interface --- cni/network/network.go | 2 +- cnm/network/network.go | 2 +- cnms/service/networkmonitor.go | 2 +- network/endpoint.go | 7 ++++--- network/endpoint_linux.go | 25 +++++++++++-------------- network/endpoint_test.go | 16 ++++++++-------- network/endpoint_windows.go | 4 ++-- network/manager.go | 8 +++++--- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index cbd9a66e25..4c0dd34e84 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -117,7 +117,7 @@ func NewPlugin(name string, nl := netlink.NewNetlink() // Setup network manager. - nm, err := network.NewNetworkManager(nl, platform.NewExecClient(logger), &netio.NetIO{}) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient(logger), &netio.NetIO{}, network.NewNamespaceClient()) if err != nil { return nil, err } diff --git a/cnm/network/network.go b/cnm/network/network.go index 6ebc132511..85a4545a11 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -53,7 +53,7 @@ func NewPlugin(config *common.PluginConfig) (NetPlugin, error) { nl := netlink.NewNetlink() // Setup network manager. - nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}, network.NewNamespaceClient()) if err != nil { return nil, err } diff --git a/cnms/service/networkmonitor.go b/cnms/service/networkmonitor.go index ff99114b20..545442c8f2 100644 --- a/cnms/service/networkmonitor.go +++ b/cnms/service/networkmonitor.go @@ -157,7 +157,7 @@ func main() { } nl := netlink.NewNetlink() - nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}) + nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}, network.NewNamespaceClient()) if err != nil { log.Printf("[monitor] Failed while creating network manager") return diff --git a/network/endpoint.go b/network/endpoint.go index 78935a3b5c..ab873e92d0 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -132,6 +132,7 @@ func (nw *network) newEndpoint( nl netlink.NetlinkInterface, plc platform.ExecClient, netioCli netio.NetIOInterface, + nsc NamespaceClientInterface, epInfo []*EndpointInfo, ) (*endpoint, error) { var ep *endpoint @@ -145,7 +146,7 @@ func (nw *network) newEndpoint( // Call the platform implementation. // Pass nil for epClient and will be initialized in newendpointImpl - ep, err = nw.newEndpointImpl(apipaCli, nl, plc, netioCli, nil, epInfo) + ep, err = nw.newEndpointImpl(apipaCli, nl, plc, netioCli, nil, nsc, epInfo) if err != nil { return nil, err } @@ -156,7 +157,7 @@ func (nw *network) newEndpoint( } // DeleteEndpoint deletes an existing endpoint from the network. -func (nw *network) deleteEndpoint(nl netlink.NetlinkInterface, plc platform.ExecClient, endpointID string) error { +func (nw *network) deleteEndpoint(nl netlink.NetlinkInterface, plc platform.ExecClient, nsc NamespaceClientInterface, endpointID string) error { var err error logger.Info("Deleting endpoint from network", zap.String("endpointID", endpointID), zap.String("id", nw.Id)) @@ -175,7 +176,7 @@ func (nw *network) deleteEndpoint(nl netlink.NetlinkInterface, plc platform.Exec // Call the platform implementation. // Pass nil for epClient and will be initialized in deleteEndpointImpl - err = nw.deleteEndpointImpl(nl, plc, nil, ep) + err = nw.deleteEndpointImpl(nl, plc, nil, nsc, ep) if err != nil { return err } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 7d45c3d61d..3ceda20460 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -55,6 +55,7 @@ func (nw *network) newEndpointImpl( plc platform.ExecClient, netioCli netio.NetIOInterface, testEpClient EndpointClient, + nsc NamespaceClientInterface, epInfo []*EndpointInfo, ) (*endpoint, error) { var ( @@ -133,7 +134,7 @@ func (nw *network) newEndpointImpl( if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { nw.SnatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) } - epClient = NewTransparentVlanEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc) + epClient = NewTransparentVlanEndpointClient(nw, epInfo, hostIfName, contIfName, vlanid, localIP, nl, plc, nsc) } else { logger.Info("OVS client") if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { @@ -156,7 +157,7 @@ func (nw *network) newEndpointImpl( epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) } else if epInfo.NICType == cns.DelegatedVMNIC { logger.Info("Secondary client") - epClient = NewSecondaryEndpointClient(nl, plc, ep) + epClient = NewSecondaryEndpointClient(nl, plc, nsc, ep) } else { logger.Info("Transparent client") epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode, nl, plc) @@ -202,7 +203,7 @@ func (nw *network) newEndpointImpl( if epInfo.NetNsPath != "" { // Open the network namespace. logger.Info("Opening netns", zap.Any("NetNsPath", epInfo.NetNsPath)) - ns, epErr := OpenNamespace(epInfo.NetNsPath) + ns, epErr := nsc.OpenNamespace(epInfo.NetNsPath) if epErr != nil { return epErr } @@ -254,7 +255,7 @@ func (nw *network) newEndpointImpl( } // deleteEndpointImpl deletes an existing endpoint from the network. -func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform.ExecClient, epClient EndpointClient, ep *endpoint) error { +func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform.ExecClient, epClient EndpointClient, nsc NamespaceClientInterface, ep *endpoint) error { // Delete the veth pair by deleting one of the peer interfaces. // Deleting the host interface is more convenient since it does not require // entering the container netns and hence works both for CNI and CNM. @@ -266,7 +267,7 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. epInfo := ep.getInfo() if nw.Mode == opModeTransparentVlan { logger.Info("Transparent vlan client") - epClient = NewTransparentVlanEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, plc) + epClient = NewTransparentVlanEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, plc, nsc) } else { epClient = NewOVSEndpointClient(nw, epInfo, ep.HostIfName, "", ep.VlanID, ep.LocalIP, nl, ovsctl.NewOvsctl(), plc) @@ -275,7 +276,7 @@ func (nw *network) deleteEndpointImpl(nl netlink.NetlinkInterface, plc platform. epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode, nl, plc) } else { if len(ep.SecondaryInterfaces) > 0 { - epClient = NewSecondaryEndpointClient(nl, plc, ep) + epClient = NewSecondaryEndpointClient(nl, plc, nsc, ep) epClient.DeleteEndpointRules(ep) //nolint:errcheck // ignore error epClient.DeleteEndpoints(ep) @@ -389,16 +390,13 @@ func deleteRoutes(nl netlink.NetlinkInterface, netioshim netio.NetIOInterface, i // updateEndpointImpl updates an existing endpoint in the network. func (nm *networkManager) updateEndpointImpl(nw *network, existingEpInfo *EndpointInfo, targetEpInfo *EndpointInfo) (*endpoint, error) { - var ns *Namespace var ep *endpoint - var err error existingEpFromRepository := nw.Endpoints[existingEpInfo.Id] logger.Info("[updateEndpointImpl] Going to retrieve endpoint with Id to update", zap.String("id", existingEpInfo.Id)) if existingEpFromRepository == nil { logger.Info("[updateEndpointImpl] Endpoint cannot be updated as it does not exist") - err = errEndpointNotFound - return nil, err + return nil, errEndpointNotFound } netns := existingEpFromRepository.NetworkNameSpace @@ -406,7 +404,7 @@ func (nm *networkManager) updateEndpointImpl(nw *network, existingEpInfo *Endpoi if netns != "" { // Open the network namespace. logger.Info("[updateEndpointImpl] Opening netns", zap.Any("netns", netns)) - ns, err = OpenNamespace(netns) + ns, err := nm.nsClient.OpenNamespace(netns) if err != nil { return nil, err } @@ -428,12 +426,11 @@ func (nm *networkManager) updateEndpointImpl(nw *network, existingEpInfo *Endpoi } else { logger.Info("[updateEndpointImpl] Endpoint cannot be updated as the network namespace does not exist: Epid", zap.String("id", existingEpInfo.Id), zap.String("component", "updateEndpointImpl")) - err = errNamespaceNotFound - return nil, err + return nil, errNamespaceNotFound } logger.Info("[updateEndpointImpl] Going to update routes in netns", zap.Any("netns", netns)) - if err = nm.updateRoutes(existingEpInfo, targetEpInfo); err != nil { + if err := nm.updateRoutes(existingEpInfo, targetEpInfo); err != nil { return nil, err } diff --git a/network/endpoint_test.go b/network/endpoint_test.go index 803e08adaf..14927125df 100644 --- a/network/endpoint_test.go +++ b/network/endpoint_test.go @@ -183,7 +183,7 @@ var _ = Describe("Test Endpoint", func() { It("Should be added", func() { // Add endpoint with valid id ep, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -195,7 +195,7 @@ var _ = Describe("Test Endpoint", func() { extIf: &externalInterface{IPv4Gateway: net.ParseIP("192.168.0.1")}, } ep, err := nw2.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) @@ -211,7 +211,7 @@ var _ = Describe("Test Endpoint", func() { Expect(err).ToNot(HaveOccurred()) // Adding endpoint with same id should fail and delete should cleanup the state ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), mockCli, []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), []*EndpointInfo{epInfo}) Expect(err).To(HaveOccurred()) Expect(ep2).To(BeNil()) assert.Contains(GinkgoT(), err.Error(), "Endpoint already exists") @@ -221,17 +221,17 @@ var _ = Describe("Test Endpoint", func() { // Adding an endpoint with an id. mockCli := NewMockEndpointClient(nil) ep2, err := nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), mockCli, []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), mockCli, NewMockNamespaceClient(), []*EndpointInfo{epInfo}) Expect(err).ToNot(HaveOccurred()) Expect(ep2).ToNot(BeNil()) Expect(len(mockCli.endpoints)).To(Equal(1)) // Deleting the endpoint //nolint:errcheck // ignore error - nw.deleteEndpointImpl(netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), mockCli, ep2) + nw.deleteEndpointImpl(netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), mockCli, NewMockNamespaceClient(), ep2) Expect(len(mockCli.endpoints)).To(Equal(0)) // Deleting same endpoint with same id should not fail //nolint:errcheck // ignore error - nw.deleteEndpointImpl(netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), mockCli, ep2) + nw.deleteEndpointImpl(netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), mockCli, NewMockNamespaceClient(), ep2) Expect(len(mockCli.endpoints)).To(Equal(0)) }) }) @@ -252,11 +252,11 @@ var _ = Describe("Test Endpoint", func() { } return nil - }), []*EndpointInfo{epInfo}) + }), NewMockNamespaceClient(), []*EndpointInfo{epInfo}) Expect(err).To(HaveOccurred()) Expect(ep).To(BeNil()) ep, err = nw.newEndpointImpl(nil, netlink.NewMockNetlink(false, ""), platform.NewMockExecClient(false), - netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), []*EndpointInfo{epInfo}) + netio.NewMockNetIO(false, 0), NewMockEndpointClient(nil), NewMockNamespaceClient(), []*EndpointInfo{epInfo}) Expect(err).NotTo(HaveOccurred()) Expect(ep).NotTo(BeNil()) Expect(ep.Id).To(Equal(epInfo.Id)) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 42c8e7af14..6e35327b73 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -64,7 +64,7 @@ func ConstructEndpointID(containerID string, netNsPath string, ifName string) (s } // newEndpointImpl creates a new endpoint in the network. -func (nw *network) newEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, plc platform.ExecClient, _ netio.NetIOInterface, _ EndpointClient, epInfo []*EndpointInfo) (*endpoint, error) { +func (nw *network) newEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, plc platform.ExecClient, _ netio.NetIOInterface, _ EndpointClient, _ NamespaceClientInterface, epInfo []*EndpointInfo) (*endpoint, error) { // there is only 1 epInfo for windows, multiple interfaces will be added in the future if useHnsV2, err := UseHnsV2(epInfo[0].NetNsPath); useHnsV2 { if err != nil { @@ -401,7 +401,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( } // deleteEndpointImpl deletes an existing endpoint from the network. -func (nw *network) deleteEndpointImpl(_ netlink.NetlinkInterface, _ platform.ExecClient, _ EndpointClient, ep *endpoint) error { +func (nw *network) deleteEndpointImpl(_ netlink.NetlinkInterface, _ platform.ExecClient, _ EndpointClient, _ NamespaceClientInterface, ep *endpoint) error { if useHnsV2, err := UseHnsV2(ep.NetNs); useHnsV2 { if err != nil { return err diff --git a/network/manager.go b/network/manager.go index 99e3b082bd..5204cc0c31 100644 --- a/network/manager.go +++ b/network/manager.go @@ -68,6 +68,7 @@ type networkManager struct { netlink netlink.NetlinkInterface netio netio.NetIOInterface plClient platform.ExecClient + nsClient NamespaceClientInterface sync.Mutex } @@ -98,12 +99,13 @@ type NetworkManager interface { } // Creates a new network manager. -func NewNetworkManager(nl netlink.NetlinkInterface, plc platform.ExecClient, netioCli netio.NetIOInterface) (NetworkManager, error) { +func NewNetworkManager(nl netlink.NetlinkInterface, plc platform.ExecClient, netioCli netio.NetIOInterface, nsc NamespaceClientInterface) (NetworkManager, error) { nm := &networkManager{ ExternalInterfaces: make(map[string]*externalInterface), netlink: nl, plClient: plc, netio: netioCli, + nsClient: nsc, } return nm, nil @@ -344,7 +346,7 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn } } - _, err = nw.newEndpoint(cli, nm.netlink, nm.plClient, nm.netio, epInfo) + _, err = nw.newEndpoint(cli, nm.netlink, nm.plClient, nm.netio, nm.nsClient, epInfo) if err != nil { return err } @@ -367,7 +369,7 @@ func (nm *networkManager) DeleteEndpoint(networkID, endpointID string) error { return err } - err = nw.deleteEndpoint(nm.netlink, nm.plClient, endpointID) + err = nw.deleteEndpoint(nm.netlink, nm.plClient, nm.nsClient, endpointID) if err != nil { return err } From 4f2de549761c0fe5d4ddb66ce8d23172b25a964c Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 20 Oct 2023 23:06:01 +0000 Subject: [PATCH 62/64] fix lint --- network/endpoint_windows.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 6e35327b73..718dc613c1 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -64,7 +64,15 @@ func ConstructEndpointID(containerID string, netNsPath string, ifName string) (s } // newEndpointImpl creates a new endpoint in the network. -func (nw *network) newEndpointImpl(cli apipaClient, _ netlink.NetlinkInterface, plc platform.ExecClient, _ netio.NetIOInterface, _ EndpointClient, _ NamespaceClientInterface, epInfo []*EndpointInfo) (*endpoint, error) { +func (nw *network) newEndpointImpl( + cli apipaClient, + _ netlink.NetlinkInterface, + plc platform.ExecClient, + _ netio.NetIOInterface, + _ EndpointClient, + _ NamespaceClientInterface, + epInfo []*EndpointInfo, +) (*endpoint, error) { // there is only 1 epInfo for windows, multiple interfaces will be added in the future if useHnsV2, err := UseHnsV2(epInfo[0].NetNsPath); useHnsV2 { if err != nil { From 3cb718b6edf595f16c53e2faa94a497b358f8632 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Mon, 23 Oct 2023 15:38:23 +0000 Subject: [PATCH 63/64] rebasing fixes --- cni/network/invoker_azure.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index f26d798f40..4e27b0bd46 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -62,7 +62,7 @@ func (invoker *AzureIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, er if err != nil && strings.Contains(err.Error(), ipam.ErrNoAvailableAddressPools.Error()) { invoker.deleteIpamState() logger.Info("Retry pool allocation after deleting IPAM state") - addResult.ipv4Result, err = invoker.plugin.DelegateAdd(addConfig.nwCfg.IPAM.Type, addConfig.nwCfg) + result, err = invoker.plugin.DelegateAdd(addConfig.nwCfg.IPAM.Type, addConfig.nwCfg) } if err != nil { From 6e14593dd5bbff734062964dee76729d362b4194 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Wed, 1 Nov 2023 15:18:16 +0000 Subject: [PATCH 64/64] address comments --- cni/network/invoker_azure_test.go | 62 +++++++++++++++++-------------- cni/network/invoker_cns.go | 16 ++++++++ 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/cni/network/invoker_azure_test.go b/cni/network/invoker_azure_test.go index 487db822bd..9bd10af61d 100644 --- a/cni/network/invoker_azure_test.go +++ b/cni/network/invoker_azure_test.go @@ -16,6 +16,14 @@ import ( "github.com/stretchr/testify/require" ) +const ( + ipv4cidr = "10.0.0.1/24" + v4NetCidr = "10.0.0.0/24" + ipv4cidr2 = "10.0.0.4/24" + ipv6cidr = "2001:0db8:abcd:0015::0/64" + v6NetCidr = "2001:db8:abcd:0012::0/64" +) + type mockDelegatePlugin struct { add del @@ -142,17 +150,17 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { fields: fields{ plugin: &mockDelegatePlugin{ add: add{ - resultsIPv4: getSingleResult("10.0.0.1/24"), + resultsIPv4: getSingleResult(ipv4cidr), }, del: del{}, }, - nwInfo: getNwInfo("10.0.0.0/24", ""), + nwInfo: getNwInfo(v4NetCidr, ""), }, args: args{ nwCfg: &cni.NetworkConfig{}, - subnetPrefix: getCIDRNotationForAddress("10.0.0.0/24"), + subnetPrefix: getCIDRNotationForAddress(v4NetCidr), }, - want: getResult("10.0.0.1/24"), + want: getResult(ipv4cidr), wantErr: false, }, { @@ -160,19 +168,19 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { fields: fields{ plugin: &mockDelegatePlugin{ add: add{ - resultsIPv4: getSingleResult("10.0.0.1/24"), - resultsIPv6: getSingleResult("2001:0db8:abcd:0015::0/64"), + resultsIPv4: getSingleResult(ipv4cidr), + resultsIPv6: getSingleResult(ipv6cidr), }, }, - nwInfo: getNwInfo("10.0.0.0/24", "2001:db8:abcd:0012::0/64"), + nwInfo: getNwInfo(v4NetCidr, v6NetCidr), }, args: args{ nwCfg: &cni.NetworkConfig{ IPV6Mode: network.IPV6Nat, }, - subnetPrefix: getCIDRNotationForAddress("10.0.0.0/24"), + subnetPrefix: getCIDRNotationForAddress(v4NetCidr), }, - want: getResult("10.0.0.1/24", "2001:0db8:abcd:0015::0/64"), + want: getResult(ipv4cidr, ipv6cidr), wantErr: false, }, { @@ -183,7 +191,7 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { errv4: errors.New("test error"), //nolint:goerr113 }, }, - nwInfo: getNwInfo("10.0.0.0/24", ""), + nwInfo: getNwInfo(v4NetCidr, ""), }, args: args{ nwCfg: &cni.NetworkConfig{}, @@ -196,19 +204,19 @@ func TestAzureIPAMInvoker_Add(t *testing.T) { fields: fields{ plugin: &mockDelegatePlugin{ add: add{ - resultsIPv4: getSingleResult("10.0.0.1/24"), + resultsIPv4: getSingleResult(ipv4cidr), errv6: errors.New("test v6 error"), //nolint:goerr113 }, }, - nwInfo: getNwInfo("10.0.0.0/24", ""), + nwInfo: getNwInfo(v4NetCidr, ""), }, args: args{ nwCfg: &cni.NetworkConfig{ IPV6Mode: network.IPV6Nat, }, - subnetPrefix: getCIDRNotationForAddress("10.0.0.0/24"), + subnetPrefix: getCIDRNotationForAddress(v4NetCidr), }, - want: getResult("10.0.0.1/24"), + want: getResult(ipv4cidr), wantErr: true, }, } @@ -260,10 +268,10 @@ func TestAzureIPAMInvoker_Delete(t *testing.T) { plugin: &mockDelegatePlugin{ del: del{}, }, - nwInfo: getNwInfo("10.0.0.0/24", ""), + nwInfo: getNwInfo(v4NetCidr, ""), }, args: args{ - address: getCIDRNotationForAddress("10.0.0.4/24"), + address: getCIDRNotationForAddress(ipv4cidr2), nwCfg: &cni.NetworkConfig{ IPAM: cni.IPAM{ Address: "10.0.0.4", @@ -277,7 +285,7 @@ func TestAzureIPAMInvoker_Delete(t *testing.T) { plugin: &mockDelegatePlugin{ del: del{}, }, - nwInfo: getNwInfo("10.0.0.0/24", "2001:db8:abcd:0012::0/64"), + nwInfo: getNwInfo(v4NetCidr, v6NetCidr), }, args: args{ address: getCIDRNotationForAddress("2001:db8:abcd:0015::0/64"), @@ -296,7 +304,7 @@ func TestAzureIPAMInvoker_Delete(t *testing.T) { err: errors.New("error when address is nil"), //nolint:goerr113 }, }, - nwInfo: getNwInfo("", "2001:db8:abcd:0012::0/64"), + nwInfo: getNwInfo("", v6NetCidr), }, args: args{ address: nil, @@ -316,13 +324,13 @@ func TestAzureIPAMInvoker_Delete(t *testing.T) { err: errors.New("error on v4 delete"), //nolint:goerr113 }, }, - nwInfo: getNwInfo("10.0.0.0/24", ""), + nwInfo: getNwInfo(v4NetCidr, ""), }, args: args{ - address: getCIDRNotationForAddress("10.0.0.4/24"), + address: getCIDRNotationForAddress(ipv4cidr2), nwCfg: &cni.NetworkConfig{ IPAM: cni.IPAM{ - Address: "10.0.0.4/24", + Address: ipv4cidr2, }, }, }, @@ -336,13 +344,13 @@ func TestAzureIPAMInvoker_Delete(t *testing.T) { err: errors.New("error on v6 delete"), //nolint:goerr113 }, }, - nwInfo: getNwInfo("10.0.0.0/24", "2001:db8:abcd:0012::0/64"), + nwInfo: getNwInfo(v4NetCidr, v6NetCidr), }, args: args{ address: getCIDRNotationForAddress("2001:db8:abcd:0015::0/64"), nwCfg: &cni.NetworkConfig{ IPAM: cni.IPAM{ - Address: "10.0.0.4/24", + Address: ipv4cidr2, }, }, }, @@ -397,17 +405,17 @@ func TestRemoveIpamState_Add(t *testing.T) { fields: fields{ plugin: &mockDelegatePlugin{ add: add{ - resultsIPv4: getSingleResult("10.0.0.1/24"), + resultsIPv4: getSingleResult(ipv4cidr), errv4: ipam.ErrNoAvailableAddressPools, }, }, - nwInfo: getNwInfo("10.0.0.0/24", ""), + nwInfo: getNwInfo(v4NetCidr, ""), }, args: args{ nwCfg: &cni.NetworkConfig{}, - subnetPrefix: getCIDRNotationForAddress("10.0.0.0/24"), + subnetPrefix: getCIDRNotationForAddress(v4NetCidr), }, - want: getResult("10.0.0.1/24"), + want: getResult(ipv4cidr), wantErrMsg: ipam.ErrNoAvailableAddressPools.Error(), wantErr: true, }, diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 674c37beb5..d8867efe85 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -19,6 +19,7 @@ import ( cniTypesCurr "github.com/containernetworking/cni/pkg/types/100" "github.com/pkg/errors" "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) const ( @@ -56,6 +57,21 @@ type IPResultInfo struct { routes []cns.Route } +func (i IPResultInfo) MarshalLogObject(encoder zapcore.ObjectEncoder) error { + encoder.AddString("podIPAddress", i.podIPAddress) + encoder.AddUint8("ncSubnetPrefix", i.ncSubnetPrefix) + encoder.AddString("ncPrimaryIP", i.ncPrimaryIP) + encoder.AddString("ncGatewayIPAddress", i.ncGatewayIPAddress) + encoder.AddString("hostSubnet", i.hostSubnet) + encoder.AddString("hostPrimaryIP", i.hostPrimaryIP) + encoder.AddString("hostGateway", i.hostGateway) + encoder.AddString("nicType", string(i.nicType)) + encoder.AddString("macAddress", i.macAddress) + encoder.AddBool("skipDefaultRoutes", i.skipDefaultRoutes) + encoder.AddString("routes", fmt.Sprintf("%+v", i.routes)) + return nil +} + func NewCNSInvoker(podName, namespace string, cnsClient cnsclient, executionMode util.ExecutionMode, ipamMode util.IpamMode) *CNSIPAMInvoker { return &CNSIPAMInvoker{ podName: podName,