diff --git a/Makefile b/Makefile index 2895bbd0e3..688b550e82 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,7 @@ CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy-tran CNI_SWIFT_BUILD_DIR = $(BUILD_DIR)/cni-swift CNI_OVERLAY_BUILD_DIR = $(BUILD_DIR)/cni-overlay CNI_BAREMETAL_BUILD_DIR = $(BUILD_DIR)/cni-baremetal +CNI_DUALSTACK_BUILD_DIR = $(BUILD_DIR)/cni-dualstack CNS_BUILD_DIR = $(BUILD_DIR)/cns NPM_BUILD_DIR = $(BUILD_DIR)/npm TOOLS_DIR = $(REPO_ROOT)/build/tools @@ -94,6 +95,7 @@ CNI_MULTITENANCY_TRANSPARENT_VLAN_ARCHIVE_NAME = azure-vnet-cni-multitenancy-tra CNI_SWIFT_ARCHIVE_NAME = azure-vnet-cni-swift-$(GOOS)-$(GOARCH)-$(CNI_VERSION).$(ARCHIVE_EXT) CNI_OVERLAY_ARCHIVE_NAME = azure-vnet-cni-overlay-$(GOOS)-$(GOARCH)-$(CNI_VERSION).$(ARCHIVE_EXT) CNI_BAREMETAL_ARCHIVE_NAME = azure-vnet-cni-baremetal-$(GOOS)-$(GOARCH)-$(CNI_VERSION).$(ARCHIVE_EXT) +CNI_DUALSTACK_ARCHIVE_NAME = azure-vnet-cni-overlay-dualstack-$(GOOS)-$(GOARCH)-$(CNI_VERSION).$(ARCHIVE_EXT) CNM_ARCHIVE_NAME = azure-vnet-cnm-$(GOOS)-$(GOARCH)-$(ACN_VERSION).$(ARCHIVE_EXT) CNS_ARCHIVE_NAME = azure-cns-$(GOOS)-$(GOARCH)-$(CNS_VERSION).$(ARCHIVE_EXT) NPM_ARCHIVE_NAME = azure-npm-$(GOOS)-$(GOARCH)-$(NPM_VERSION).$(ARCHIVE_EXT) @@ -624,6 +626,12 @@ endif cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_OVERLAY_BUILD_DIR) cd $(CNI_OVERLAY_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_OVERLAY_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config + $(MKDIR) $(CNI_DUALSTACK_BUILD_DIR) + cp cni/azure-$(GOOS)-swift-overlay-dualstack.conflist $(CNI_DUALSTACK_BUILD_DIR)/10-azure.conflist + cp telemetry/azure-vnet-telemetry.config $(CNI_DUALSTACK_BUILD_DIR)/azure-vnet-telemetry.config + cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_DUALSTACK_BUILD_DIR) + cd $(CNI_DUALSTACK_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_DUALSTACK_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config + #baremetal mode is windows only (at least for now) ifeq ($(GOOS),windows) $(MKDIR) $(CNI_BAREMETAL_BUILD_DIR) diff --git a/cni/azure-linux-swift-overlay-dualstack.conflist b/cni/azure-linux-swift-overlay-dualstack.conflist new file mode 100644 index 0000000000..92bd31a628 --- /dev/null +++ b/cni/azure-linux-swift-overlay-dualstack.conflist @@ -0,0 +1,22 @@ +{ + "cniVersion":"0.3.0", + "name":"azure", + "plugins":[ + { + "type":"azure-vnet", + "mode":"transparent", + "ipsToRouteViaHost":["169.254.20.10"], + "ipam":{ + "type":"azure-cns", + "mode":"dualStackOverlay" + } + }, + { + "type":"portmap", + "capabilities":{ + "portMappings":true + }, + "snat":true + } + ] +} diff --git a/cni/azure-windows-swift-overlay-dualstack.conflist b/cni/azure-windows-swift-overlay-dualstack.conflist new file mode 100644 index 0000000000..c84e56d19b --- /dev/null +++ b/cni/azure-windows-swift-overlay-dualstack.conflist @@ -0,0 +1,49 @@ +{ + "cniVersion": "0.3.0", + "name": "azure", + "adapterName" : "", + "plugins": [ + { + "type": "azure-vnet", + "mode": "bridge", + "bridge": "azure0", + "capabilities": { + "portMappings": true, + "dns": true + }, + "ipam": { + "type": "azure-cns", + "mode": "dualStackOverlay" + }, + "dns": { + "Nameservers": [ + "10.0.0.10", + "168.63.129.16" + ], + "Search": [ + "svc.cluster.local" + ] + }, + "AdditionalArgs": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "ExceptionList": [ + "10.240.0.0/16", + "10.0.0.0/8" + ] + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ROUTE", + "DestinationPrefix": "10.0.0.0/8", + "NeedEncap": true + } + } + ] + } + ] +} diff --git a/cni/network/cnsclient.go b/cni/network/cnsclient.go index ea26cae02c..b48a2a2b4d 100644 --- a/cni/network/cnsclient.go +++ b/cni/network/cnsclient.go @@ -9,6 +9,8 @@ import ( type cnsclient interface { RequestIPAddress(ctx context.Context, ipconfig cns.IPConfigRequest) (*cns.IPConfigResponse, error) ReleaseIPAddress(ctx context.Context, ipconfig cns.IPConfigRequest) error + RequestIPs(ctx context.Context, ipconfig cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) + ReleaseIPs(ctx context.Context, ipconfig cns.IPConfigsRequest) error GetNetworkContainer(ctx context.Context, orchestratorContext []byte) (*cns.GetNetworkContainerResponse, error) GetAllNetworkContainers(ctx context.Context, orchestratorContext []byte) ([]cns.GetNetworkContainerResponse, error) } diff --git a/cni/network/invoker_azure.go b/cni/network/invoker_azure.go index c0f85c8baa..360aab0f76 100644 --- a/cni/network/invoker_azure.go +++ b/cni/network/invoker_azure.go @@ -18,6 +18,11 @@ import ( cniTypesCurr "github.com/containernetworking/cni/pkg/types/100" ) +const ( + bytesSize4 = 4 + bytesSize16 = 16 +) + type AzureIPAMInvoker struct { plugin delegatePlugin nwInfo *network.NetworkInfo @@ -122,7 +127,7 @@ func (invoker *AzureIPAMInvoker) deleteIpamState() { } } -func (invoker *AzureIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConfig, _ *cniSkel.CmdArgs, options map[string]interface{}) error { +func (invoker *AzureIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConfig, _ *cniSkel.CmdArgs, options map[string]interface{}) error { //nolint if nwCfg == nil { return invoker.plugin.Errorf("nil nwCfg passed to CNI ADD, stack: %+v", string(debug.Stack())) } @@ -135,25 +140,28 @@ func (invoker *AzureIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkCo if err := invoker.plugin.DelegateDel(nwCfg.IPAM.Type, nwCfg); err != nil { return invoker.plugin.Errorf("Attempted to release address with error: %v", err) } - } else if len(address.IP.To4()) == 4 { + } else if len(address.IP.To4()) == bytesSize4 { //nolint:gocritic nwCfg.IPAM.Address = address.IP.String() - log.Printf("Releasing ipv4 address :%s pool: %s", - nwCfg.IPAM.Address, nwCfg.IPAM.Subnet) + log.Printf("Releasing ipv4 address :%s pool: %s", nwCfg.IPAM.Address, nwCfg.IPAM.Subnet) if err := invoker.plugin.DelegateDel(nwCfg.IPAM.Type, nwCfg); err != nil { log.Printf("Failed to release ipv4 address: %v", err) - return invoker.plugin.Errorf("Failed to release ipv4 address: %v", err) + return invoker.plugin.Errorf("Failed to release ipv4 address: %v with error: ", nwCfg.IPAM.Address, err) } - } else if len(address.IP.To16()) == 16 { + } else if len(address.IP.To16()) == bytesSize16 { nwCfgIpv6 := *nwCfg nwCfgIpv6.IPAM.Environment = common.OptEnvironmentIPv6NodeIpam nwCfgIpv6.IPAM.Type = ipamV6 nwCfgIpv6.IPAM.Address = address.IP.String() if len(invoker.nwInfo.Subnets) > 1 { - nwCfgIpv6.IPAM.Subnet = invoker.nwInfo.Subnets[1].Prefix.String() + for _, subnet := range invoker.nwInfo.Subnets { + if subnet.Prefix.IP.To4() == nil { + nwCfgIpv6.IPAM.Subnet = subnet.Prefix.String() + break + } + } } - log.Printf("Releasing ipv6 address :%s pool: %s", - nwCfgIpv6.IPAM.Address, nwCfgIpv6.IPAM.Subnet) + log.Printf("Releasing ipv6 address :%s pool: %s", nwCfgIpv6.IPAM.Address, nwCfgIpv6.IPAM.Subnet) if err := invoker.plugin.DelegateDel(nwCfgIpv6.IPAM.Type, &nwCfgIpv6); err != nil { log.Printf("Failed to release ipv6 address: %v", err) return invoker.plugin.Errorf("Failed to release ipv6 address: %v", err) diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index c484824d7e..08b9a088ab 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cni/util" "github.com/Azure/azure-container-networking/cns" + cnscli "github.com/Azure/azure-container-networking/cns/client" "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" @@ -20,8 +21,9 @@ import ( ) var ( - errEmptyCNIArgs = errors.New("empty CNI cmd args not allowed") - errInvalidArgs = errors.New("invalid arg(s)") + errEmptyCNIArgs = errors.New("empty CNI cmd args not allowed") + errInvalidArgs = errors.New("invalid arg(s)") + overlayGatewayV6IP = "fe80::1234:5678:9abc" ) type CNSIPAMInvoker struct { @@ -32,7 +34,7 @@ type CNSIPAMInvoker struct { ipamMode util.IpamMode } -type IPv4ResultInfo struct { +type IPResultInfo struct { podIPAddress string ncSubnetPrefix uint8 ncPrimaryIP string @@ -70,94 +72,144 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro return IPAMAddResult{}, errEmptyCNIArgs } - ipconfig := cns.IPConfigRequest{ + ipconfigs := cns.IPConfigsRequest{ OrchestratorContext: orchestratorContext, PodInterfaceID: GetEndpointID(addConfig.args), InfraContainerID: addConfig.args.ContainerID, } - log.Printf("Requesting IP for pod %+v using ipconfig %+v", podInfo, ipconfig) - response, err := invoker.cnsClient.RequestIPAddress(context.TODO(), ipconfig) + log.Printf("Requesting IP for pod %+v using ipconfigs %+v", podInfo, ipconfigs) + response, err := invoker.cnsClient.RequestIPs(context.TODO(), ipconfigs) if err != nil { - log.Printf("Failed to get IP address from CNS with error %v, response: %v", err, response) - return IPAMAddResult{}, errors.Wrap(err, "Failed to get IP address from CNS with error: %w") - } - - info := IPv4ResultInfo{ - podIPAddress: response.PodIpInfo.PodIPConfig.IPAddress, - ncSubnetPrefix: response.PodIpInfo.NetworkContainerPrimaryIPConfig.IPSubnet.PrefixLength, - ncPrimaryIP: response.PodIpInfo.NetworkContainerPrimaryIPConfig.IPSubnet.IPAddress, - ncGatewayIPAddress: response.PodIpInfo.NetworkContainerPrimaryIPConfig.GatewayIPAddress, - hostSubnet: response.PodIpInfo.HostPrimaryIPInfo.Subnet, - hostPrimaryIP: response.PodIpInfo.HostPrimaryIPInfo.PrimaryIP, - hostGateway: response.PodIpInfo.HostPrimaryIPInfo.Gateway, + if cnscli.IsUnsupportedAPI(err) { + // If RequestIPs is not supported by CNS, use RequestIPAddress API + log.Errorf("RequestIPs not supported by CNS. Invoking RequestIPAddress API with infracontainerid %s", ipconfigs.InfraContainerID) + ipconfig := cns.IPConfigRequest{ + OrchestratorContext: orchestratorContext, + PodInterfaceID: GetEndpointID(addConfig.args), + InfraContainerID: addConfig.args.ContainerID, + } + + res, errRequestIP := invoker.cnsClient.RequestIPAddress(context.TODO(), ipconfig) + if errRequestIP != nil { + // if the old API fails as well then we just return the error + log.Errorf("Failed to request IP address from CNS using RequestIPAddress with infracontainerid %s. error: %v", ipconfig.InfraContainerID, errRequestIP) + return IPAMAddResult{}, errors.Wrap(errRequestIP, "Failed to get IP address from CNS with error: %w") + } + response = &cns.IPConfigsResponse{ + Response: res.Response, + PodIPInfo: []cns.PodIpInfo{ + res.PodIpInfo, + }, + } + } else { + log.Printf("Failed to get IP address from CNS with error %v, response: %v", err, response) + return IPAMAddResult{}, errors.Wrap(err, "Failed to get IP address from CNS with error: %w") + } } - // set the NC Primary IP in options - addConfig.options[network.SNATIPKey] = info.ncPrimaryIP + addResult := IPAMAddResult{} - log.Printf("[cni-invoker-cns] Received info %+v for pod %v", info, podInfo) + for i := 0; i < len(response.PodIPInfo); i++ { + info := IPResultInfo{ + podIPAddress: response.PodIPInfo[i].PodIPConfig.IPAddress, + ncSubnetPrefix: response.PodIPInfo[i].NetworkContainerPrimaryIPConfig.IPSubnet.PrefixLength, + ncPrimaryIP: response.PodIPInfo[i].NetworkContainerPrimaryIPConfig.IPSubnet.IPAddress, + ncGatewayIPAddress: response.PodIPInfo[i].NetworkContainerPrimaryIPConfig.GatewayIPAddress, + hostSubnet: response.PodIPInfo[i].HostPrimaryIPInfo.Subnet, + hostPrimaryIP: response.PodIPInfo[i].HostPrimaryIPInfo.PrimaryIP, + hostGateway: response.PodIPInfo[i].HostPrimaryIPInfo.Gateway, + } - // set result ipconfigArgument from CNS Response Body - ip, ncipnet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix)) - if ip == nil { - return IPAMAddResult{}, errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") - } + // set the NC Primary IP in options + // SNATIPKey is not set for ipv6 + if net.ParseIP(info.ncPrimaryIP).To4() != nil { + addConfig.options[network.SNATIPKey] = info.ncPrimaryIP + } - ncgw := net.ParseIP(info.ncGatewayIPAddress) - if ncgw == nil { - if invoker.ipamMode != util.V4Overlay { - return IPAMAddResult{}, errors.Wrap(errInvalidArgs, "%w: Gateway address "+info.ncGatewayIPAddress+" from response is invalid") + log.Printf("[cni-invoker-cns] Received info %+v for pod %v", info, podInfo) + ip, ncIPNet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix)) + if ip == nil { + return IPAMAddResult{}, errors.Wrap(err, "Unable to parse IP from response: "+info.podIPAddress+" with err %w") } - ncgw, err = getOverlayGateway(ncipnet) - if err != nil { - return IPAMAddResult{}, err + ncgw := net.ParseIP(info.ncGatewayIPAddress) + if ncgw == nil { + if (invoker.ipamMode != util.V4Overlay) && (invoker.ipamMode != util.DualStackOverlay) { + return IPAMAddResult{}, errors.Wrap(errInvalidArgs, "%w: Gateway address "+info.ncGatewayIPAddress+" from response is invalid") + } + + if net.ParseIP(info.podIPAddress).To4() != nil { //nolint:gocritic + ncgw, err = getOverlayGateway(ncIPNet) + if err != nil { + return IPAMAddResult{}, err + } + } else if net.ParseIP(info.podIPAddress).To16() != nil { + ncgw = net.ParseIP(overlayGatewayV6IP) + } else { + return IPAMAddResult{}, errors.Wrap(err, "No podIPAddress is found: %w") + } } - } - // construct ipnet for result - resultIPnet := net.IPNet{ - IP: ip, - Mask: ncipnet.Mask, - } + // construct ipnet for result + resultIPnet := net.IPNet{ + IP: ip, + Mask: ncIPNet.Mask, + } - addResult := IPAMAddResult{} - addResult.ipv4Result = &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { - Address: resultIPnet, - Gateway: ncgw, - }, - }, - Routes: []*cniTypes.Route{ - { - Dst: network.Ipv4DefaultRouteDstPrefix, - GW: ncgw, - }, - }, - } + if net.ParseIP(info.podIPAddress).To4() != nil { + addResult.ipv4Result = &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: resultIPnet, + Gateway: ncgw, + }, + }, + Routes: []*cniTypes.Route{ + { + Dst: network.Ipv4DefaultRouteDstPrefix, + GW: ncgw, + }, + }, + } + } else if net.ParseIP(info.podIPAddress).To16() != nil { + addResult.ipv6Result = &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: resultIPnet, + Gateway: ncgw, + }, + }, + Routes: []*cniTypes.Route{ + { + Dst: network.Ipv6DefaultRouteDstPrefix, + GW: ncgw, + }, + }, + } + } - // get the name of the primary IP address - _, hostIPNet, err := net.ParseCIDR(info.hostSubnet) - if err != nil { - return IPAMAddResult{}, fmt.Errorf("unable to parse hostSubnet: %w", err) - } + // get the name of the primary IP address + _, hostIPNet, err := net.ParseCIDR(info.hostSubnet) + if err != nil { + return IPAMAddResult{}, fmt.Errorf("unable to parse hostSubnet: %w", err) + } - addResult.hostSubnetPrefix = *hostIPNet + addResult.hostSubnetPrefix = *hostIPNet - // set subnet prefix for host vm - // setHostOptions will execute if IPAM mode is not v4 overlay - if invoker.ipamMode != util.V4Overlay { - if err := setHostOptions(ncipnet, addConfig.options, &info); err != nil { - return IPAMAddResult{}, err + // set subnet prefix for host vm + // setHostOptions will execute if IPAM mode is not v4 overlay and not dualStackOverlay mode + if (invoker.ipamMode != util.V4Overlay) && (invoker.ipamMode != util.DualStackOverlay) { + if err := setHostOptions(ncIPNet, addConfig.options, &info); err != nil { + return IPAMAddResult{}, err + } } } return addResult, nil } -func setHostOptions(ncSubnetPrefix *net.IPNet, options map[string]interface{}, info *IPv4ResultInfo) error { +func setHostOptions(ncSubnetPrefix *net.IPNet, options map[string]interface{}, info *IPResultInfo) error { // get the host ip hostIP := net.ParseIP(info.hostPrimaryIP) if hostIP == nil { @@ -213,7 +265,7 @@ func setHostOptions(ncSubnetPrefix *net.IPNet, options map[string]interface{}, i } // Delete calls into the releaseipconfiguration API in CNS -func (invoker *CNSIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConfig, args *cniSkel.CmdArgs, _ map[string]interface{}) error { +func (invoker *CNSIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConfig, args *cniSkel.CmdArgs, _ map[string]interface{}) error { //nolint // Parse Pod arguments. podInfo := cns.KubernetesPodInfo{ PodName: invoker.podName, @@ -229,20 +281,37 @@ func (invoker *CNSIPAMInvoker) Delete(address *net.IPNet, nwCfg *cni.NetworkConf return errEmptyCNIArgs } - req := cns.IPConfigRequest{ + ipConfigs := cns.IPConfigsRequest{ OrchestratorContext: orchestratorContext, PodInterfaceID: GetEndpointID(args), InfraContainerID: args.ContainerID, } if address != nil { - req.DesiredIPAddress = address.IP.String() + ipConfigs.DesiredIPAddresses = append(ipConfigs.DesiredIPAddresses, address.IP.String()) } else { log.Printf("CNS invoker called with empty IP address") } - if err := invoker.cnsClient.ReleaseIPAddress(context.TODO(), req); err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to release IP %v with err ", address)+"%w") + if err := invoker.cnsClient.ReleaseIPs(context.TODO(), ipConfigs); err != nil { + if cnscli.IsUnsupportedAPI(err) { + // If ReleaseIPs is not supported by CNS, use ReleaseIPAddress API + log.Errorf("ReleaseIPs not supported by CNS. Invoking ReleaseIPAddress API. Request: %v", ipConfigs) + ipConfig := cns.IPConfigRequest{ + OrchestratorContext: orchestratorContext, + PodInterfaceID: GetEndpointID(args), + InfraContainerID: args.ContainerID, + } + + if err = invoker.cnsClient.ReleaseIPAddress(context.TODO(), ipConfig); err != nil { + // if the old API fails as well then we just return the error + log.Errorf("Failed to release IP address from CNS using ReleaseIPAddress with infracontainerid %s. error: %v", ipConfigs.InfraContainerID, err) + return errors.Wrap(err, fmt.Sprintf("failed to release IP %v using ReleaseIPAddress with err ", ipConfig.DesiredIPAddress)+"%w") + } + } else { + log.Errorf("Failed to release IP address with infracontainerid %s from CNS error: %v", ipConfigs.InfraContainerID, err) + return errors.Wrap(err, fmt.Sprintf("failed to release IP %v using ReleaseIPs with err ", ipConfigs.DesiredIPAddresses)+"%w") + } } return nil diff --git a/cni/network/invoker_cns_test.go b/cni/network/invoker_cns_test.go index 1506fa176f..9d3325440a 100644 --- a/cni/network/invoker_cns_test.go +++ b/cni/network/invoker_cns_test.go @@ -28,6 +28,14 @@ func getTestIPConfigRequest() cns.IPConfigRequest { } } +func getTestIPConfigsRequest() cns.IPConfigsRequest { + return cns.IPConfigsRequest{ + PodInterfaceID: "testcont-testifname", + InfraContainerID: "testcontainerid", + OrchestratorContext: marshallPodInfo(testPodInfo), + } +} + func getTestOverlayGateway() net.IP { if runtime.GOOS == "windows" { return net.ParseIP("10.240.0.1") @@ -36,8 +44,13 @@ func getTestOverlayGateway() net.IP { return net.ParseIP("169.254.1.1") } -func TestCNSIPAMInvoker_Add(t *testing.T) { +func TestCNSIPAMInvoker_Add_Overlay(t *testing.T) { require := require.New(t) //nolint further usage of require without passing t + + // set new CNS API is not supported + unsupportedAPIs := make(map[cnsAPIName]struct{}) + unsupportedAPIs["RequestIPs"] = struct{}{} + type fields struct { podName string podNamespace string @@ -52,40 +65,46 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { } tests := []struct { - name string - fields fields - args args - want *cniTypesCurr.Result - want1 *cniTypesCurr.Result - wantErr bool + name string + fields fields + args args + wantIpv4Result *cniTypesCurr.Result + wantIpv6Result *cniTypesCurr.Result + wantErr bool }{ { - name: "Test happy CNI add", + name: "Test happy CNI Overlay add in v4overlay ipamMode", fields: fields{ podName: testPodInfo.PodName, podNamespace: testPodInfo.PodNamespace, + ipamMode: util.V4Overlay, cnsClient: &MockCNSClient{ - require: require, - request: requestIPAddressHandler{ - ipconfigArgument: getTestIPConfigRequest(), + unsupportedAPIs: unsupportedAPIs, + require: require, + requestIP: requestIPAddressHandler{ + ipconfigArgument: cns.IPConfigRequest{ + PodInterfaceID: "testcont-testifname3", + InfraContainerID: "testcontainerid3", + OrchestratorContext: marshallPodInfo(testPodInfo), + }, result: &cns.IPConfigResponse{ PodIpInfo: cns.PodIpInfo{ PodIPConfig: cns.IPSubnet{ - IPAddress: "10.0.1.10", - PrefixLength: 24, + IPAddress: "10.240.1.242", + PrefixLength: 16, }, NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ IPSubnet: cns.IPSubnet{ - IPAddress: "10.0.1.0", - PrefixLength: 24, + IPAddress: "10.240.1.0", + PrefixLength: 16, }, DNSServers: nil, - GatewayIPAddress: "10.0.0.1", + GatewayIPAddress: "", }, HostPrimaryIPInfo: cns.HostIPInfo{ - Gateway: "10.0.0.1", - PrimaryIP: "10.0.0.1", - Subnet: "10.0.0.0/24", + Gateway: "10.224.0.1", + PrimaryIP: "10.224.0.5", + Subnet: "10.224.0.0/16", }, }, Response: cns.Response{ @@ -100,78 +119,78 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { args: args{ nwCfg: &cni.NetworkConfig{}, args: &cniSkel.CmdArgs{ - ContainerID: "testcontainerid", - Netns: "testnetns", - IfName: "testifname", + ContainerID: "testcontainerid3", + Netns: "testnetns3", + IfName: "testifname3", }, - hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), + hostSubnetPrefix: getCIDRNotationForAddress("10.224.0.0/16"), options: map[string]interface{}{}, }, - want: &cniTypesCurr.Result{ + wantIpv4Result: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { - Address: *getCIDRNotationForAddress("10.0.1.10/24"), - Gateway: net.ParseIP("10.0.0.1"), + Address: *getCIDRNotationForAddress("10.240.1.242/16"), + Gateway: getTestOverlayGateway(), }, }, Routes: []*cniTypes.Route{ { Dst: network.Ipv4DefaultRouteDstPrefix, - GW: net.ParseIP("10.0.0.1"), - }, - }, - }, - want1: nil, - wantErr: false, - }, - { - name: "fail to request IP address from cns", - fields: fields{ - podName: testPodInfo.PodName, - podNamespace: testPodInfo.PodNamespace, - cnsClient: &MockCNSClient{ - require: require, - request: requestIPAddressHandler{ - ipconfigArgument: getTestIPConfigRequest(), - result: nil, - err: errors.New("failed error from CNS"), //nolint "error for ut" + GW: getTestOverlayGateway(), }, }, }, - wantErr: true, + wantIpv6Result: nil, + wantErr: false, }, { - name: "Test happy CNI Overlay add", + name: "Test happy CNI Overlay add in dualstack overlay ipamMode", fields: fields{ podName: testPodInfo.PodName, podNamespace: testPodInfo.PodNamespace, - ipamMode: util.V4Overlay, cnsClient: &MockCNSClient{ require: require, - request: requestIPAddressHandler{ - ipconfigArgument: cns.IPConfigRequest{ - PodInterfaceID: "testcont-testifname3", - InfraContainerID: "testcontainerid3", - OrchestratorContext: marshallPodInfo(testPodInfo), - }, - result: &cns.IPConfigResponse{ - PodIpInfo: cns.PodIpInfo{ - PodIPConfig: cns.IPSubnet{ - IPAddress: "10.240.1.242", - PrefixLength: 16, - }, - NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ - IPSubnet: cns.IPSubnet{ - IPAddress: "10.240.1.0", - PrefixLength: 16, + requestIPs: requestIPsHandler{ + ipconfigArgument: getTestIPConfigsRequest(), + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + DNSServers: nil, + GatewayIPAddress: "10.0.0.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", }, - DNSServers: nil, - GatewayIPAddress: "", }, - HostPrimaryIPInfo: cns.HostIPInfo{ - Gateway: "10.224.0.1", - PrimaryIP: "10.224.0.5", - Subnet: "10.224.0.0/16", + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "fd11:1234::1", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "fd11:1234::", + PrefixLength: 112, + }, + DNSServers: nil, + GatewayIPAddress: "fe80::1234:5678:9abc", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "fe80::1234:5678:9abc", + PrimaryIP: "fe80::1234:5678:9abc", + Subnet: "fd11:1234::/112", + }, }, }, Response: cns.Response{ @@ -186,28 +205,41 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { args: args{ nwCfg: &cni.NetworkConfig{}, args: &cniSkel.CmdArgs{ - ContainerID: "testcontainerid3", - Netns: "testnetns3", - IfName: "testifname3", + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", }, - hostSubnetPrefix: getCIDRNotationForAddress("10.224.0.0/16"), + hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), options: map[string]interface{}{}, }, - want: &cniTypesCurr.Result{ + wantIpv4Result: &cniTypesCurr.Result{ IPs: []*cniTypesCurr.IPConfig{ { - Address: *getCIDRNotationForAddress("10.240.1.242/16"), - Gateway: getTestOverlayGateway(), + Address: *getCIDRNotationForAddress("10.0.1.10/24"), + Gateway: net.ParseIP("10.0.0.1"), }, }, Routes: []*cniTypes.Route{ { Dst: network.Ipv4DefaultRouteDstPrefix, - GW: getTestOverlayGateway(), + GW: net.ParseIP("10.0.0.1"), + }, + }, + }, + wantIpv6Result: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: *getCIDRNotationForAddress("fd11:1234::1/112"), + Gateway: net.ParseIP("fe80::1234:5678:9abc"), + }, + }, + Routes: []*cniTypes.Route{ + { + Dst: network.Ipv6DefaultRouteDstPrefix, + GW: net.ParseIP("fe80::1234:5678:9abc"), }, }, }, - want1: nil, wantErr: false, }, } @@ -229,63 +261,213 @@ func TestCNSIPAMInvoker_Add(t *testing.T) { require.NoError(err) } - fmt.Printf("want:%+v\nrest:%+v\n", tt.want, ipamAddResult.ipv4Result) - require.Equalf(tt.want, ipamAddResult.ipv4Result, "incorrect ipv4 response") - require.Equalf(tt.want1, ipamAddResult.ipv6Result, "incorrect ipv6 response") + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantIpv4Result, ipamAddResult.ipv4Result) + require.Equalf(tt.wantIpv4Result, ipamAddResult.ipv4Result, "incorrect ipv4 response") + require.Equalf(tt.wantIpv6Result, ipamAddResult.ipv6Result, "incorrect ipv6 response") }) } } -func TestCNSIPAMInvoker_Delete(t *testing.T) { +func TestCNSIPAMInvoker_Add(t *testing.T) { require := require.New(t) //nolint further usage of require without passing t type fields struct { podName string podNamespace string cnsClient cnsclient + ipamMode util.IpamMode } type args struct { - address *net.IPNet - nwCfg *cni.NetworkConfig - args *cniSkel.CmdArgs - options map[string]interface{} + nwCfg *cni.NetworkConfig + args *cniSkel.CmdArgs + hostSubnetPrefix *net.IPNet + options map[string]interface{} } + tests := []struct { - name string - fields fields - args args - wantErr bool + name string + fields fields + args args + wantIpv4Result *cniTypesCurr.Result + wantIpv6Result *cniTypesCurr.Result + wantErr bool }{ { - name: "test delete happy path", + name: "Test happy CNI add", fields: fields{ podName: testPodInfo.PodName, podNamespace: testPodInfo.PodNamespace, cnsClient: &MockCNSClient{ require: require, - release: releaseIPAddressHandler{ - ipconfigArgument: getTestIPConfigRequest(), + requestIPs: requestIPsHandler{ + ipconfigArgument: getTestIPConfigsRequest(), + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + DNSServers: nil, + GatewayIPAddress: "10.0.0.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", + }, + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, }, }, }, args: args{ - nwCfg: nil, + nwCfg: &cni.NetworkConfig{}, args: &cniSkel.CmdArgs{ ContainerID: "testcontainerid", Netns: "testnetns", IfName: "testifname", }, - options: map[string]interface{}{}, + hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), + options: map[string]interface{}{}, }, + wantIpv4Result: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: *getCIDRNotationForAddress("10.0.1.10/24"), + Gateway: net.ParseIP("10.0.0.1"), + }, + }, + Routes: []*cniTypes.Route{ + { + Dst: network.Ipv4DefaultRouteDstPrefix, + GW: net.ParseIP("10.0.0.1"), + }, + }, + }, + wantIpv6Result: nil, + wantErr: false, }, { - name: "test delete not happy path", + name: "Test happy CNI add for both ipv4 and ipv6", fields: fields{ podName: testPodInfo.PodName, podNamespace: testPodInfo.PodNamespace, cnsClient: &MockCNSClient{ - release: releaseIPAddressHandler{ - ipconfigArgument: getTestIPConfigRequest(), - err: errors.New("handle CNS delete error"), //nolint ut error + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: getTestIPConfigsRequest(), + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + DNSServers: nil, + GatewayIPAddress: "10.0.0.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", + }, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "fd11:1234::1", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "fd11:1234::", + PrefixLength: 112, + }, + DNSServers: nil, + GatewayIPAddress: "fe80::1234:5678:9abc", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "fe80::1234:5678:9abc", + PrimaryIP: "fe80::1234:5678:9abc", + Subnet: "fd11:1234::/112", + }, + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), + options: map[string]interface{}{}, + }, + wantIpv4Result: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: *getCIDRNotationForAddress("10.0.1.10/24"), + Gateway: net.ParseIP("10.0.0.1"), + }, + }, + Routes: []*cniTypes.Route{ + { + Dst: network.Ipv4DefaultRouteDstPrefix, + GW: net.ParseIP("10.0.0.1"), + }, + }, + }, + wantIpv6Result: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: *getCIDRNotationForAddress("fd11:1234::1/112"), + Gateway: net.ParseIP("fe80::1234:5678:9abc"), + }, + }, + Routes: []*cniTypes.Route{ + { + Dst: network.Ipv6DefaultRouteDstPrefix, + GW: net.ParseIP("fe80::1234:5678:9abc"), + }, + }, + }, + wantErr: false, + }, + { + name: "fail to request IP addresses from cns", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: getTestIPConfigsRequest(), + result: nil, + err: errors.New("failed error from CNS"), //nolint "error for ut" }, }, }, @@ -300,12 +482,539 @@ func TestCNSIPAMInvoker_Delete(t *testing.T) { podNamespace: tt.fields.podNamespace, cnsClient: tt.fields.cnsClient, } - err := invoker.Delete(tt.args.address, tt.args.nwCfg, tt.args.args, tt.args.options) + if tt.fields.ipamMode != "" { + invoker.ipamMode = tt.fields.ipamMode + } + ipamAddResult, err := invoker.Add(IPAMAddConfig{nwCfg: tt.args.nwCfg, args: tt.args.args, options: tt.args.options}) if tt.wantErr { require.Error(err) } else { require.NoError(err) } + + fmt.Printf("want:%+v\nrest:%+v\n", tt.wantIpv4Result, ipamAddResult.ipv4Result) + require.Equalf(tt.wantIpv4Result, ipamAddResult.ipv4Result, "incorrect ipv4 response") + require.Equalf(tt.wantIpv6Result, ipamAddResult.ipv6Result, "incorrect ipv6 response") + }) + } +} + +func TestCNSIPAMInvoker_Add_UnsupportedAPI(t *testing.T) { + require := require.New(t) //nolint further usage of require without passing t + + // set new CNS API is not supported + unsupportedAPIs := make(map[cnsAPIName]struct{}) + unsupportedAPIs["RequestIPs"] = struct{}{} + + type fields struct { + podName string + podNamespace string + cnsClient cnsclient + ipamMode util.IpamMode + } + type args struct { + nwCfg *cni.NetworkConfig + args *cniSkel.CmdArgs + hostSubnetPrefix *net.IPNet + options map[string]interface{} + } + + tests := []struct { + name string + fields fields + args args + want *cniTypesCurr.Result + want1 *cniTypesCurr.Result + wantErr bool + }{ + { + name: "Test happy CNI add for IPv4 without RequestIPs supported", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + unsupportedAPIs: unsupportedAPIs, + require: require, + requestIP: requestIPAddressHandler{ + ipconfigArgument: getTestIPConfigRequest(), + result: &cns.IPConfigResponse{ + PodIpInfo: cns.PodIpInfo{ + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + DNSServers: nil, + GatewayIPAddress: "10.0.0.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), + options: map[string]interface{}{}, + }, + want: &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Address: *getCIDRNotationForAddress("10.0.1.10/24"), + Gateway: net.ParseIP("10.0.0.1"), + }, + }, + Routes: []*cniTypes.Route{ + { + Dst: network.Ipv4DefaultRouteDstPrefix, + GW: net.ParseIP("10.0.0.1"), + }, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + invoker := &CNSIPAMInvoker{ + podName: tt.fields.podName, + podNamespace: tt.fields.podNamespace, + cnsClient: tt.fields.cnsClient, + } + if tt.fields.ipamMode != "" { + invoker.ipamMode = tt.fields.ipamMode + } + ipamAddResult, err := invoker.Add(IPAMAddConfig{nwCfg: tt.args.nwCfg, args: tt.args.args, options: tt.args.options}) + if err != nil && tt.wantErr { + t.Fatalf("expected an error %+v but none received", err) + } + require.NoError(err) + require.Equalf(tt.want, ipamAddResult.ipv4Result, "incorrect ipv4 response") + }) + } +} + +func TestRequestIPAPIsFail(t *testing.T) { + require := require.New(t) //nolint further usage of require without passing t + + type fields struct { + podName string + podNamespace string + cnsClient cnsclient + ipamMode util.IpamMode + } + type args struct { + nwCfg *cni.NetworkConfig + args *cniSkel.CmdArgs + hostSubnetPrefix *net.IPNet + options map[string]interface{} + } + + tests := []struct { + name string + fields fields + args args + want *cniTypesCurr.Result + want1 *cniTypesCurr.Result + wantErr bool + }{ + { + name: "Test happy CNI add for dualstack mode with both requestIP and requestIPs get failed", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + require: require, + requestIPs: requestIPsHandler{ + ipconfigArgument: getTestIPConfigsRequest(), + result: &cns.IPConfigsResponse{ + PodIPInfo: []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 24, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "10.0.1.0", + PrefixLength: 24, + }, + DNSServers: nil, + GatewayIPAddress: "10.0.0.1", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "10.0.0.1", + PrimaryIP: "10.0.0.1", + Subnet: "10.0.0.0/24", + }, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "fd11:1234::1", + PrefixLength: 112, + }, + NetworkContainerPrimaryIPConfig: cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "fd11:1234::", + PrefixLength: 112, + }, + DNSServers: nil, + GatewayIPAddress: "fe80::1234:5678:9abc", + }, + HostPrimaryIPInfo: cns.HostIPInfo{ + Gateway: "fe80::1234:5678:9abc", + PrimaryIP: "fe80::1234:5678:9abc", + Subnet: "fd11:1234::/112", + }, + }, + }, + Response: cns.Response{ + ReturnCode: 0, + Message: "", + }, + }, + err: nil, + }, + }, + }, + args: args{ + nwCfg: &cni.NetworkConfig{}, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns1", + IfName: "testifname1", + }, + hostSubnetPrefix: getCIDRNotationForAddress("10.0.0.1/24"), + options: map[string]interface{}{}, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + invoker := &CNSIPAMInvoker{ + podName: tt.fields.podName, + podNamespace: tt.fields.podNamespace, + cnsClient: tt.fields.cnsClient, + } + if tt.fields.ipamMode != "" { + invoker.ipamMode = tt.fields.ipamMode + } + _, err := invoker.Add(IPAMAddConfig{nwCfg: tt.args.nwCfg, args: tt.args.args, options: tt.args.options}) + if err == nil && tt.wantErr { + t.Fatalf("expected an error %+v but none received", err) + } + if !errors.Is(err, errNoRequestIPFound) { + t.Fatalf("expected an error %s but %v received", errNoRequestIPFound, err) + } + }) + } +} + +func TestCNSIPAMInvoker_Delete(t *testing.T) { + require := require.New(t) //nolint further usage of require without passing t + type fields struct { + podName string + podNamespace string + cnsClient cnsclient + } + type args struct { + address *net.IPNet + nwCfg *cni.NetworkConfig + args *cniSkel.CmdArgs + options map[string]interface{} + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "test delete happy path", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + require: require, + releaseIPs: releaseIPsHandler{ + ipconfigArgument: getTestIPConfigsRequest(), + }, + }, + }, + args: args{ + nwCfg: nil, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + options: map[string]interface{}{}, + }, + }, + { + name: "test delete not happy path", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + releaseIPs: releaseIPsHandler{ + ipconfigArgument: getTestIPConfigsRequest(), + err: errors.New("handle CNS delete error"), //nolint ut error + }, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + invoker := &CNSIPAMInvoker{ + podName: tt.fields.podName, + podNamespace: tt.fields.podNamespace, + cnsClient: tt.fields.cnsClient, + } + err := invoker.Delete(tt.args.address, tt.args.nwCfg, tt.args.args, tt.args.options) + if tt.wantErr { + require.Error(err) + } else { + require.NoError(err) + } + }) + } +} + +func TestCNSIPAMInvoker_Delete_Overlay(t *testing.T) { + require := require.New(t) //nolint further usage of require without passing t + + // set new CNS API is not supported + unsupportedAPIs := make(map[cnsAPIName]struct{}) + unsupportedAPIs["ReleaseIPs"] = struct{}{} + + type fields struct { + podName string + podNamespace string + cnsClient cnsclient + ipamMode util.IpamMode + } + type args struct { + address *net.IPNet + nwCfg *cni.NetworkConfig + args *cniSkel.CmdArgs + options map[string]interface{} + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "test delete happy path in v4overlay ipamMode", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + ipamMode: util.V4Overlay, + cnsClient: &MockCNSClient{ + unsupportedAPIs: unsupportedAPIs, + require: require, + releaseIP: releaseIPHandler{ + ipconfigArgument: getTestIPConfigRequest(), + }, + }, + }, + args: args{ + nwCfg: nil, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + options: map[string]interface{}{}, + }, + }, + { + name: "test delete happy path in dualStackOverlay ipamMode", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + ipamMode: util.DualStackOverlay, + cnsClient: &MockCNSClient{ + require: require, + releaseIPs: releaseIPsHandler{ + ipconfigArgument: getTestIPConfigsRequest(), + }, + }, + }, + args: args{ + nwCfg: nil, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + options: map[string]interface{}{}, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + invoker := &CNSIPAMInvoker{ + podName: tt.fields.podName, + podNamespace: tt.fields.podNamespace, + cnsClient: tt.fields.cnsClient, + } + err := invoker.Delete(tt.args.address, tt.args.nwCfg, tt.args.args, tt.args.options) + if tt.wantErr { + require.Error(err) + } else { + require.NoError(err) + } + }) + } +} + +func TestCNSIPAMInvoker_Delete_NotSupportedAPI(t *testing.T) { + require := require.New(t) //nolint further usage of require without passing t + // set new CNS API is not supported + unsupportedAPIs := make(map[cnsAPIName]struct{}) + unsupportedAPIs["ReleaseIPs"] = struct{}{} + + type fields struct { + podName string + podNamespace string + cnsClient cnsclient + } + type args struct { + address *net.IPNet + nwCfg *cni.NetworkConfig + args *cniSkel.CmdArgs + options map[string]interface{} + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "test delete happy path with unsupportedAPI", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + unsupportedAPIs: unsupportedAPIs, + require: require, + releaseIP: releaseIPHandler{ + ipconfigArgument: getTestIPConfigRequest(), + }, + }, + }, + args: args{ + nwCfg: nil, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns", + IfName: "testifname", + }, + options: map[string]interface{}{}, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + invoker := &CNSIPAMInvoker{ + podName: tt.fields.podName, + podNamespace: tt.fields.podNamespace, + cnsClient: tt.fields.cnsClient, + } + err := invoker.Delete(tt.args.address, tt.args.nwCfg, tt.args.args, tt.args.options) + if tt.wantErr { + require.Error(err) + } else { + require.NoError(err) + } + }) + } +} + +func TestReleaseIPAPIsFail(t *testing.T) { + require := require.New(t) //nolint further usage of require without passing t + type fields struct { + podName string + podNamespace string + cnsClient cnsclient + } + type args struct { + address *net.IPNet + nwCfg *cni.NetworkConfig + args *cniSkel.CmdArgs + options map[string]interface{} + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "test delete with both cns releaseIPs and releaseIP get failed", + fields: fields{ + podName: testPodInfo.PodName, + podNamespace: testPodInfo.PodNamespace, + cnsClient: &MockCNSClient{ + require: require, + releaseIPs: releaseIPsHandler{ + ipconfigArgument: getTestIPConfigsRequest(), + }, + }, + }, + args: args{ + nwCfg: nil, + args: &cniSkel.CmdArgs{ + ContainerID: "testcontainerid", + Netns: "testnetns1", + IfName: "testifname1", + }, + options: map[string]interface{}{}, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + invoker := &CNSIPAMInvoker{ + podName: tt.fields.podName, + podNamespace: tt.fields.podNamespace, + cnsClient: tt.fields.cnsClient, + } + err := invoker.Delete(tt.args.address, tt.args.nwCfg, tt.args.args, tt.args.options) + if !errors.Is(err, errNoReleaseIPFound) { + t.Fatalf("expected an error %s but %v received", errNoReleaseIPFound, err) + } }) } } @@ -316,7 +1025,7 @@ func Test_setHostOptions(t *testing.T) { hostSubnetPrefix *net.IPNet ncSubnetPrefix *net.IPNet options map[string]interface{} - info IPv4ResultInfo + info IPResultInfo } tests := []struct { name string @@ -330,7 +1039,7 @@ func Test_setHostOptions(t *testing.T) { hostSubnetPrefix: getCIDRNotationForAddress("10.0.1.0/24"), ncSubnetPrefix: getCIDRNotationForAddress("10.0.1.0/24"), options: map[string]interface{}{}, - info: IPv4ResultInfo{ + info: IPResultInfo{ podIPAddress: "10.0.1.10", ncSubnetPrefix: 24, ncPrimaryIP: "10.0.1.20", @@ -376,7 +1085,7 @@ func Test_setHostOptions(t *testing.T) { { name: "test error on bad host subnet", args: args{ - info: IPv4ResultInfo{ + info: IPResultInfo{ hostSubnet: "", }, }, @@ -385,7 +1094,7 @@ func Test_setHostOptions(t *testing.T) { { name: "test error on nil hostsubnetprefix", args: args{ - info: IPv4ResultInfo{ + info: IPResultInfo{ hostSubnet: "10.0.0.0/24", }, }, diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 8c5f5fdcf5..242d01254e 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -300,30 +300,6 @@ func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse, ifName s return result } -func getInfraVnetIP( - enableInfraVnet bool, - infraSubnet string, - nwCfg *cni.NetworkConfig, - plugin *NetPlugin, -) (*cniTypesCurr.Result, error) { - if enableInfraVnet { - _, ipNet, _ := net.ParseCIDR(infraSubnet) - nwCfg.IPAM.Subnet = ipNet.String() - - log.Printf("call ipam to allocate ip from subnet %v", nwCfg.IPAM.Subnet) - ipamAddOpt := IPAMAddConfig{nwCfg: nwCfg, options: make(map[string]interface{})} - ipamAddResult, err := plugin.ipamInvoker.Add(ipamAddOpt) - if err != nil { - err = plugin.Errorf("Failed to allocate address: %v", err) - return nil, err - } - - return ipamAddResult.ipv4Result, nil - } - - return nil, nil -} - func checkIfSubnetOverlaps(enableInfraVnet bool, nwCfg *cni.NetworkConfig, cnsNetworkConfig *cns.GetNetworkContainerResponse) bool { if enableInfraVnet { if cnsNetworkConfig != nil { diff --git a/cni/network/multitenancy_test.go b/cni/network/multitenancy_test.go index 6e65f5994d..d98f96234d 100644 --- a/cni/network/multitenancy_test.go +++ b/cni/network/multitenancy_test.go @@ -28,9 +28,29 @@ type requestIPAddressHandler struct { err error } -type releaseIPAddressHandler struct { +type requestIPsHandler struct { + // arguments + ipconfigArgument cns.IPConfigsRequest + + // results + result *cns.IPConfigsResponse // this will return the IPConfigsResponse which contains a slice of IPs as opposed to one IP + err error +} + +type releaseIPHandler struct { + // arguments ipconfigArgument cns.IPConfigRequest - err error + + // results + err error +} + +type releaseIPsHandler struct { + // arguments + ipconfigArgument cns.IPConfigsRequest + + // results + err error } type getNetworkContainerConfigurationHandler struct { @@ -50,30 +70,68 @@ type cnsAPIName string const ( GetAllNetworkContainers cnsAPIName = "GetAllNetworkContainers" + RequestIPs cnsAPIName = "RequestIPs" + ReleaseIPs cnsAPIName = "ReleaseIPs" ) var ( errUnsupportedAPI = errors.New("Unsupported API") + errNoRequestIPFound = errors.New("No Request IP Found") + errNoReleaseIPFound = errors.New("No Release IP Found") errNoOrchestratorContextFound = errors.New("No CNI OrchestratorContext Found") ) type MockCNSClient struct { unsupportedAPIs map[cnsAPIName]struct{} require *require.Assertions - request requestIPAddressHandler - release releaseIPAddressHandler + requestIP requestIPAddressHandler + requestIPs requestIPsHandler + releaseIP releaseIPHandler + releaseIPs releaseIPsHandler getNetworkContainerConfiguration getNetworkContainerConfigurationHandler getAllNetworkContainersConfiguration getAllNetworkContainersConfigurationHandler } func (c *MockCNSClient) RequestIPAddress(_ context.Context, ipconfig cns.IPConfigRequest) (*cns.IPConfigResponse, error) { - c.require.Exactly(c.request.ipconfigArgument, ipconfig) - return c.request.result, c.request.err + if !cmp.Equal(c.requestIP.ipconfigArgument, ipconfig) { + return nil, errNoRequestIPFound + } + return c.requestIP.result, c.requestIP.err +} + +func (c *MockCNSClient) RequestIPs(_ context.Context, ipconfig cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { + if _, isUnsupported := c.unsupportedAPIs[RequestIPs]; isUnsupported { + e := &client.CNSClientError{} + e.Code = types.UnsupportedAPI + e.Err = errUnsupportedAPI + return nil, e + } + + if !cmp.Equal(c.requestIPs.ipconfigArgument, ipconfig) { + return nil, errNoRequestIPFound + } + return c.requestIPs.result, c.requestIPs.err } func (c *MockCNSClient) ReleaseIPAddress(_ context.Context, ipconfig cns.IPConfigRequest) error { - c.require.Exactly(c.release.ipconfigArgument, ipconfig) - return c.release.err + if !cmp.Equal(c.releaseIP.ipconfigArgument, ipconfig) { + return errNoReleaseIPFound + } + return c.releaseIP.err +} + +func (c *MockCNSClient) ReleaseIPs(_ context.Context, ipconfig cns.IPConfigsRequest) error { + if _, isUnsupported := c.unsupportedAPIs[ReleaseIPs]; isUnsupported { + e := &client.CNSClientError{} + e.Code = types.UnsupportedAPI + e.Err = errUnsupportedAPI + return e + } + + if !cmp.Equal(c.releaseIPs.ipconfigArgument, ipconfig) { + return errNoReleaseIPFound + } + return c.releaseIPs.err } func (c *MockCNSClient) GetNetworkContainer(ctx context.Context, orchestratorContext []byte) (*cns.GetNetworkContainerResponse, error) { diff --git a/cni/network/network.go b/cni/network/network.go index ca4929858f..6c1542d0cb 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -372,6 +372,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } addSnatInterface(nwCfg, ipamAddResult.ipv4Result) + // Convert result to the requested CNI version. res, vererr := ipamAddResult.ipv4Result.GetAsVersion(nwCfg.CNIVersion) if vererr != nil { @@ -572,6 +573,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { return nil } +// cleanup allocated ipv4 and ipv6 addresses if they exist func (plugin *NetPlugin) cleanupAllocationOnError( result, resultV6 *cniTypesCurr.Result, nwCfg *cni.NetworkConfig, @@ -622,25 +624,12 @@ func (plugin *NetPlugin) createNetworkInternal( log.Printf("[cni-net] nwDNSInfo: %v", nwDNSInfo) - var podSubnetPrefix *net.IPNet - _, podSubnetPrefix, err = net.ParseCIDR(ipamAddResult.ipv4Result.IPs[0].Address.String()) - if err != nil { - return nwInfo, fmt.Errorf("Failed to ParseCIDR for pod subnet prefix: %w", err) - } - // Create the network. nwInfo = network.NetworkInfo{ - Id: networkID, - Mode: ipamAddConfig.nwCfg.Mode, - MasterIfName: masterIfName, - AdapterName: ipamAddConfig.nwCfg.AdapterName, - Subnets: []network.SubnetInfo{ - { - Family: platform.AfINET, - Prefix: *podSubnetPrefix, - Gateway: ipamAddResult.ipv4Result.IPs[0].Gateway, - }, - }, + Id: networkID, + Mode: ipamAddConfig.nwCfg.Mode, + MasterIfName: masterIfName, + AdapterName: ipamAddConfig.nwCfg.AdapterName, BridgeName: ipamAddConfig.nwCfg.Bridge, EnableSnatOnHost: ipamAddConfig.nwCfg.EnableSnatOnHost, DNS: nwDNSInfo, @@ -653,10 +642,12 @@ func (plugin *NetPlugin) createNetworkInternal( ServiceCidrs: ipamAddConfig.nwCfg.ServiceCidrs, } + if nwInfo, err = addSubnetToNetworkInfo(ipamAddResult, nwInfo); err != nil { + log.Printf("[cni-net] Failed to add subnets to networkInfo due to %+v", err) + return nwInfo, err + } setNetworkOptions(ipamAddResult.ncResponse, &nwInfo) - addNatIPV6SubnetInfo(ipamAddConfig.nwCfg, ipamAddResult.ipv6Result, &nwInfo) - err = plugin.nm.CreateNetwork(&nwInfo) if err != nil { err = plugin.Errorf("createNetworkInternal: Failed to create network: %v", err) @@ -665,6 +656,43 @@ func (plugin *NetPlugin) createNetworkInternal( return nwInfo, err } +// construct network info with ipv4/ipv6 subnets +func addSubnetToNetworkInfo(ipamAddResult IPAMAddResult, nwInfo network.NetworkInfo) (network.NetworkInfo, error) { + var ( + podSubnetPrefix *net.IPNet + podSubnetV6Prefix *net.IPNet + ) + + _, podSubnetPrefix, err := net.ParseCIDR(ipamAddResult.ipv4Result.IPs[0].Address.String()) + if err != nil { + return network.NetworkInfo{}, fmt.Errorf("Failed to ParseCIDR for pod subnet prefix: %w", err) + } + + ipv4Subnet := network.SubnetInfo{ + Family: platform.AfINET, + Prefix: *podSubnetPrefix, + Gateway: ipamAddResult.ipv4Result.IPs[0].Gateway, + } + nwInfo.Subnets = append(nwInfo.Subnets, ipv4Subnet) + + // parse the ipv6 address and only add it to nwInfo if it's dual stack mode + if ipamAddResult.ipv6Result != nil && len(ipamAddResult.ipv6Result.IPs) > 0 { + _, podSubnetV6Prefix, err = net.ParseCIDR(ipamAddResult.ipv6Result.IPs[0].Address.String()) + if err != nil { + return network.NetworkInfo{}, fmt.Errorf("Failed to ParseCIDR for pod subnet IPv6 prefix: %w", err) + } + + ipv6Subnet := network.SubnetInfo{ + Family: platform.AfINET6, + Prefix: *podSubnetV6Prefix, + Gateway: ipamAddResult.ipv6Result.IPs[0].Gateway, + } + nwInfo.Subnets = append(nwInfo.Subnets, ipv6Subnet) + } + + return nwInfo, nil +} + type createEndpointInternalOpt struct { nwCfg *cni.NetworkConfig cnsNetworkConfig *cns.GetNetworkContainerResponse @@ -742,6 +770,13 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) } if opt.resultV6 != nil { + // inject ipv6 routes to Linux pod if ipamMode is dual stack overlay + ipamMode := string(util.IpamMode(opt.nwCfg.IPAM.Mode)) + if ipamMode == string(util.DualStackOverlay) { + epInfo.IPV6Mode = ipamMode + } + log.Printf("current ipv6 mode is %s", epInfo.IPV6Mode) + for _, ipconfig := range opt.resultV6.IPs { epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) } diff --git a/cni/network/network_test.go b/cni/network/network_test.go index 7a9edc5bcb..9d72757557 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -70,13 +70,14 @@ func TestMain(m *testing.M) { func GetTestResources() *NetPlugin { pluginName := "testplugin" + isIPv6 := false config := &common.PluginConfig{} grpcClient := &nns.MockGrpcClient{} plugin, _ := NewPlugin(pluginName, config, grpcClient, &Multitenancy{}) plugin.report = &telemetry.CNIReport{} mockNetworkManager := acnnetwork.NewMockNetworkmanager() plugin.nm = mockNetworkManager - plugin.ipamInvoker = NewMockIpamInvoker(false, false, false) + plugin.ipamInvoker = NewMockIpamInvoker(isIPv6, false, false) return plugin } diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index b080046565..a0de632838 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -379,7 +379,7 @@ func determineWinVer() { } func getNATInfo(nwCfg *cni.NetworkConfig, ncPrimaryIPIface interface{}, enableSnatForDNS bool) (natInfo []policy.NATInfo) { - if nwCfg.ExecutionMode == string(util.V4Swift) && nwCfg.IPAM.Mode != string(util.V4Overlay) { + if nwCfg.ExecutionMode == string(util.V4Swift) && nwCfg.IPAM.Mode != string(util.V4Overlay) && nwCfg.IPAM.Mode != string(util.DualStackOverlay) { ncPrimaryIP := "" if ncPrimaryIPIface != nil { ncPrimaryIP = ncPrimaryIPIface.(string) diff --git a/cni/util/const.go b/cni/util/const.go index 76a28e8ac7..d3f88b2499 100644 --- a/cni/util/const.go +++ b/cni/util/const.go @@ -13,5 +13,6 @@ type IpamMode string // IPAM modes const ( - V4Overlay IpamMode = "v4overlay" + V4Overlay IpamMode = "v4overlay" + DualStackOverlay IpamMode = "dualStackOverlay" ) diff --git a/cns/client/client.go b/cns/client/client.go index d3fe3d9a94..16b85710e2 100644 --- a/cns/client/client.go +++ b/cns/client/client.go @@ -400,7 +400,6 @@ func (c *Client) RequestIPs(ctx context.Context, ipconfig cns.IPConfigsRequest) } req.Header.Set(headerContentType, contentTypeJSON) res, err := c.client.Do(req) - if err != nil { return nil, errors.Wrap(err, "http request failed") } diff --git a/dropgz/build/cniTest.Dockerfile b/dropgz/build/cniTest.Dockerfile index 0ee38de5ab..bad909886e 100644 --- a/dropgz/build/cniTest.Dockerfile +++ b/dropgz/build/cniTest.Dockerfile @@ -20,6 +20,7 @@ COPY --from=azure-ipam /azure-ipam/*.conflist pkg/embed/fs COPY --from=azure-ipam /azure-ipam/bin/* pkg/embed/fs COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift.conflist pkg/embed/fs/azure-swift.conflist COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift-overlay.conflist pkg/embed/fs/azure-swift-overlay.conflist +COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift-overlay-dualstack.conflist pkg/embed/fs/azure-swift-overlay-dualstack.conflist COPY --from=azure-vnet /azure-container-networking/bin/* pkg/embed/fs RUN cd pkg/embed/fs/ && sha256sum * > sum.txt RUN gzip --verbose --best --recursive pkg/embed/fs && for f in pkg/embed/fs/*.gz; do mv -- "$f" "${f%%.gz}"; done diff --git a/dropgz/build/linux.Dockerfile b/dropgz/build/linux.Dockerfile index 741ef6f65f..c39ac6a876 100644 --- a/dropgz/build/linux.Dockerfile +++ b/dropgz/build/linux.Dockerfile @@ -29,6 +29,7 @@ COPY --from=azure-ipam /azure-ipam/azure-ipam pkg/embed/fs COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS.conflist pkg/embed/fs/azure.conflist COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift.conflist pkg/embed/fs/azure-swift.conflist COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift-overlay.conflist pkg/embed/fs/azure-swift-overlay.conflist +COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift-overlay-dualstack.conflist pkg/embed/fs/azure-swift-overlay-dualstack.conflist COPY --from=azure-vnet /azure-container-networking/azure-vnet pkg/embed/fs COPY --from=azure-vnet /azure-container-networking/azure-vnet-telemetry pkg/embed/fs COPY --from=azure-vnet /azure-container-networking/azure-vnet-ipam pkg/embed/fs diff --git a/network/manager.go b/network/manager.go index e640feb464..683610a44c 100644 --- a/network/manager.go +++ b/network/manager.go @@ -19,13 +19,14 @@ import ( const ( // Network store key. - storeKey = "Network" - VlanIDKey = "VlanID" - AzureCNS = "azure-cns" - SNATIPKey = "NCPrimaryIPKey" - RoutesKey = "RoutesKey" - IPTablesKey = "IPTablesKey" - genericData = "com.docker.network.generic" + storeKey = "Network" + VlanIDKey = "VlanID" + AzureCNS = "azure-cns" + SNATIPKey = "NCPrimaryIPKey" + RoutesKey = "RoutesKey" + IPTablesKey = "IPTablesKey" + genericData = "com.docker.network.generic" + ipv6AddressMask = 128 ) var Ipv4DefaultRouteDstPrefix = net.IPNet{ @@ -33,6 +34,12 @@ var Ipv4DefaultRouteDstPrefix = net.IPNet{ Mask: net.IPv4Mask(0, 0, 0, 0), } +var Ipv6DefaultRouteDstPrefix = net.IPNet{ + IP: net.IPv6zero, + // This mask corresponds to a /0 subnet for IPv6 + Mask: net.CIDRMask(0, ipv6AddressMask), +} + type NetworkClient interface { CreateBridge() error DeleteBridge() error diff --git a/network/network_windows.go b/network/network_windows.go index b440e01842..ddc8049a4c 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -7,13 +7,14 @@ import ( "encoding/json" "errors" "fmt" + "net" "strconv" "strings" "time" - "github.com/Azure/azure-container-networking/network/hnswrapper" - + "github.com/Azure/azure-container-networking/cni/util" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/hnswrapper" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" @@ -31,10 +32,14 @@ const ( defaultRouteCIDR = "0.0.0.0/0" // prefix for interface name created by azure network ifNamePrefix = "vEthernet" + // ipv4 default hop + ipv4DefaultHop = "0.0.0.0" // ipv6 default hop ipv6DefaultHop = "::" // ipv6 route cmd routeCmd = "netsh interface ipv6 %s route \"%s\" \"%s\" \"%s\" store=persistent" + // add ipv4 and ipv6 route rules to windows node + netRouteCmd = "netsh interface %s add route \"%s\" \"%s\" \"%s\"" ) // Windows implementation of route. @@ -185,6 +190,60 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern return nw, nil } +// add ipv4 and ipv6 routes (if dual stack mode) to windows Node +// in dualstack mode, pods are created from different subnets on different nodes, gateway has to be node ip if pods want to communicate with each other +// add routes to make node understand pod IPs come from different subnets and VFP will take decisions based on these routes to forward traffic and avoid Natting +func (nm *networkManager) addNewNetRules(nwInfo *NetworkInfo) error { + var ( + err error + out string + ) + + // get interface name of VM adapter + ifName := nwInfo.MasterIfName + if !strings.Contains(nwInfo.MasterIfName, ifNamePrefix) { + ifName = fmt.Sprintf("%s (%s)", ifNamePrefix, nwInfo.MasterIfName) + } + + // iterate subnet and add ipv4 and ipv6 default route and gateway only if it is not existing + for _, subnet := range nwInfo.Subnets { + prefix := subnet.Prefix.String() + gateway := subnet.Gateway.String() + + ip, _, errParseCIDR := net.ParseCIDR(prefix) + if errParseCIDR != nil { + return fmt.Errorf("[net] failed to parse prefix %s due to %+v", prefix, errParseCIDR) // nolint + } + if ip.To4() != nil { + // netsh interface ipv4 add route $subnetV4 $hostInterfaceAlias "0.0.0.0" + netshV4DefaultRoute := fmt.Sprintf(netRouteCmd, "ipv4", prefix, ifName, ipv4DefaultHop) + if out, err = nm.plClient.ExecuteCommand(netshV4DefaultRoute); err != nil { + log.Printf("[net] Adding ipv4 default route failed: %v:%v", out, err) + } + + // netsh interface ipv4 add route $subnetV4 $hostInterfaceAlias $gatewayV4 + netshV4GatewayRoute := fmt.Sprintf(netRouteCmd, "ipv4", prefix, ifName, gateway) + if out, err = nm.plClient.ExecuteCommand(netshV4GatewayRoute); err != nil { + log.Printf("[net] Adding ipv4 gateway route failed: %v:%v", out, err) + } + } else { + // netsh interface ipv6 add route $subnetV6 $hostInterfaceAlias "::" + netshV6DefaultRoute := fmt.Sprintf(netRouteCmd, "ipv6", prefix, ifName, ipv6DefaultHop) + if out, err = nm.plClient.ExecuteCommand(netshV6DefaultRoute); err != nil { + log.Printf("[net] Adding ipv6 default route failed: %v:%v", out, err) + } + + // netsh interface ipv6 add route $subnetV6 $hostInterfaceAlias $gatewayV6 + netshV6GatewayRoute := fmt.Sprintf(netRouteCmd, "ipv6", prefix, ifName, gateway) + if out, err = nm.plClient.ExecuteCommand(netshV6GatewayRoute); err != nil { + log.Printf("[net] Adding ipv6 gateway route failed: %v:%v", out, err) + } + } + } + + return err // nolint +} + func (nm *networkManager) appIPV6RouteEntry(nwInfo *NetworkInfo) error { var ( err error @@ -329,6 +388,14 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern if err != nil { // if network not found, create the HNS network. if errors.As(err, &hcn.NetworkNotFoundError{}) { + // in dualStackOverlay mode, need to add net routes to windows node + // check if it is dualStackOverlay mode + if nwInfo.IPV6Mode == string(util.DualStackOverlay) { + if err := nm.addNewNetRules(nwInfo); err != nil { // nolint + log.Printf("[net] Failed to add net rules due to %+v", err) + return nil, err + } + } log.Printf("[net] Creating hcn network: %+v", hcnNetwork) hnsResponse, err = Hnsv2.CreateNetwork(hcnNetwork) diff --git a/network/transparent_endpointclient_linux.go b/network/transparent_endpointclient_linux.go index 013515af51..5fe3dcdf9f 100644 --- a/network/transparent_endpointclient_linux.go +++ b/network/transparent_endpointclient_linux.go @@ -262,6 +262,8 @@ func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(e return fmt.Errorf("Adding arp in container failed: %w", err) } + // IPv6Mode can be ipv6NAT or dual stack overlay + // set epInfo ipv6Mode to 'dualStackOverlay' to set ipv6Routes and ipv6NeighborEntries for Linux pod in dualstackOverlay ipam mode if epInfo.IPV6Mode != "" { if err := client.setupIPV6Routes(); err != nil { return err