From 9dd980f5c1fbe848dece608578f63a8c8637407a Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Tue, 28 Sep 2021 14:45:23 -0700 Subject: [PATCH 1/2] Add DNS NAT policy for windows AKS-Swift scenario --- cni/network/network.go | 16 ++++++++++++-- network/endpoint.go | 1 + network/endpoint_windows.go | 3 ++- network/policy/policy.go | 5 +++++ network/policy/policy_windows.go | 37 ++++++++++++++++++++++++++------ 5 files changed, 53 insertions(+), 9 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 6b0b9def05..4d621d0d32 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -505,6 +505,10 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // Allocate from azure ipam if !nwCfg.MultiTenancy { + if nwCfg.Ipam.Type == network.AzureCNS { + enableSnatForDns = true + } + result, resultV6, err = plugin.ipamInvoker.Add(nwCfg, args, &subnetPrefix, options) if err != nil { return err @@ -529,8 +533,13 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Created network %v with subnet %v.", networkID, subnetPrefix.String()) } + ncPrimaryIP, exists := options[network.SNATIPKey] + if !exists { + ncPrimaryIP = "" + } + epInfo, err := plugin.createEndpointInternal(nwCfg, cnsNetworkConfig, result, resultV6, azIpamResult, args, &nwInfo, - policies, endpointId, k8sPodName, k8sNamespace, enableInfraVnet, enableSnatForDns) + policies, endpointId, k8sPodName, k8sNamespace, ncPrimaryIP.(string), enableInfraVnet, enableSnatForDns) if err != nil { log.Errorf("Endpoint creation failed:%w", err) return err @@ -668,6 +677,7 @@ func (plugin *NetPlugin) createEndpointInternal( endpointID string, k8sPodName string, k8sNamespace string, + ncPrimaryIP string, enableInfraVnet bool, enableSnatForDNS bool, ) (network.EndpointInfo, error) { @@ -711,6 +721,7 @@ func (plugin *NetPlugin) createEndpointInternal( EnableMultiTenancy: nwCfg.MultiTenancy, EnableInfraVnet: enableInfraVnet, EnableSnatForDns: enableSnatForDNS, + IsAksSwift: nwCfg.Ipam.Type == network.AzureCNS && !nwCfg.MultiTenancy, PODName: k8sPodName, PODNameSpace: k8sNamespace, SkipHotAttachEp: false, // Hot attach at the time of endpoint creation @@ -720,9 +731,10 @@ func (plugin *NetPlugin) createEndpointInternal( } epPolicies := getPoliciesFromRuntimeCfg(nwCfg) - epInfo.Policies = append(epInfo.Policies, epPolicies...) + epInfo.Data[policy.NcPrimaryIPKey] = ncPrimaryIP + // Populate addresses. for _, ipconfig := range result.IPs { epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) diff --git a/network/endpoint.go b/network/endpoint.go index f54363e1bb..ee09833c39 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -69,6 +69,7 @@ type EndpointInfo struct { EnableInfraVnet bool EnableMultiTenancy bool EnableSnatForDns bool + IsAksSwift bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool NetworkContainerID string diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index ea1d51381e..7050ad3245 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -249,7 +249,8 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE MacAddress: epInfo.MacAddress.String(), } - if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy); err == nil { + if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data, + epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy, epInfo.IsAksSwift); err == nil { for _, epPolicy := range endpointPolicies { hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) } diff --git a/network/policy/policy.go b/network/policy/policy.go index 6a176c62c5..e1a4fc3e45 100644 --- a/network/policy/policy.go +++ b/network/policy/policy.go @@ -4,6 +4,11 @@ import ( "encoding/json" ) +const ( + // NcPrimaryIPKey indicates constant for the key string + NcPrimaryIPKey string = "NCPrimaryIPKey" +) + const ( NetworkPolicy CNIPolicyType = "NetworkPolicy" EndpointPolicy CNIPolicyType = "EndpointPolicy" diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go index 4fe2830dec..b7482a41af 100644 --- a/network/policy/policy_windows.go +++ b/network/policy/policy_windows.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" + "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" @@ -427,10 +428,10 @@ func GetHcnL4WFPProxyPolicy(policy Policy) (hcn.EndpointPolicy, error) { } // GetHcnEndpointPolicies returns array of all endpoint policies. -func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}, enableSnatForDns, enableMultiTenancy bool) ([]hcn.EndpointPolicy, error) { +func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}, enableSnatForDns, enableMultiTenancy, isAksSwift bool) ([]hcn.EndpointPolicy, error) { var ( hcnEndPointPolicies []hcn.EndpointPolicy - snatAndSerialize = enableMultiTenancy && enableSnatForDns + snatAndSerialize = (enableMultiTenancy && enableSnatForDns) || isAksSwift ) for _, policy := range policies { if policy.Type == policyType { @@ -468,7 +469,20 @@ func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoD } if snatAndSerialize && ValidWinVerForDnsNat { - dnsNatPolicy, err := AddDnsNATPolicyV2() + var snatToIP string + if isAksSwift { + snatToIP = epInfoData[NcPrimaryIPKey].(string) + + imdsNatPolicy, err := AddImdsNATPolicyV2() + if err != nil { + log.Printf("Failed to retrieve ImdsNAT endpoint policy due to error: %v", err) + return hcnEndPointPolicies, err + } + + hcnEndPointPolicies = append(hcnEndPointPolicies, imdsNatPolicy) + } + + dnsNatPolicy, err := AddDnsNATPolicyV2(snatToIP) if err != nil { log.Printf("Failed to retrieve DnsNAT endpoint policy due to error: %v", err) return hcnEndPointPolicies, err @@ -484,15 +498,26 @@ func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoD func AddDnsNATPolicyV1() (json.RawMessage, error) { outBoundNatPolicy := hcsshim.OutboundNatPolicy{ Policy: hcsshim.Policy{Type: hcsshim.OutboundNat}, - Destinations: []string{"168.63.129.16"}, + Destinations: []string{iptables.AzureDNS}, } serializedPolicy, err := json.Marshal(outBoundNatPolicy) return serializedPolicy, err } // AddDnsNATPolicyV2 returns DNS NAT endpoint policy for HNSv2 -func AddDnsNATPolicyV2() (hcn.EndpointPolicy, error) { - outBoundNatPolicySettings := hcn.OutboundNatPolicySetting{Destinations: []string{"168.63.129.16"}} +func AddDnsNATPolicyV2(snatToIP string) (hcn.EndpointPolicy, error) { + outBoundNatPolicySettings := hcn.OutboundNatPolicySetting{VirtualIP: snatToIP, Destinations: []string{iptables.AzureDNS}} + outBoundNatPolicySettingsBytes, err := json.Marshal(outBoundNatPolicySettings) + endpointPolicy := hcn.EndpointPolicy{ + Type: hcn.OutBoundNAT, + Settings: outBoundNatPolicySettingsBytes, + } + return endpointPolicy, err +} + +// AddImdsNATPolicyV2 returns IMDS NAT endpoint policy for HNSv2 +func AddImdsNATPolicyV2() (hcn.EndpointPolicy, error) { + outBoundNatPolicySettings := hcn.OutboundNatPolicySetting{Destinations: []string{iptables.AzureIMDS}} outBoundNatPolicySettingsBytes, err := json.Marshal(outBoundNatPolicySettings) endpointPolicy := hcn.EndpointPolicy{ Type: hcn.OutBoundNAT, From e13596981ed441a6773e09d0cd67ccfb8b844c90 Mon Sep 17 00:00:00 2001 From: Jaeryn Date: Fri, 15 Oct 2021 15:39:04 -0700 Subject: [PATCH 2/2] Addressing comments --- cni/network/invoker_cns.go | 5 +- cni/network/network.go | 151 ++++++++++++++------------- cni/network/network_linux.go | 4 + cni/network/network_test.go | 113 +++++++++++++++++++- cni/network/network_windows.go | 17 +++ cni/util/const.go | 10 ++ iptables/iptables.go | 6 -- network/endpoint.go | 2 +- network/endpoint_windows.go | 3 +- network/networkutils/networkutils.go | 7 ++ network/policy/policy.go | 11 +- network/policy/policy_windows.go | 51 +++------ 12 files changed, 249 insertions(+), 131 deletions(-) create mode 100644 cni/util/const.go create mode 100644 network/networkutils/networkutils.go diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index a31356e0ac..b201984d2c 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -12,6 +12,7 @@ import ( "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" + "github.com/Azure/azure-container-networking/network/networkutils" cniSkel "github.com/containernetworking/cni/pkg/skel" cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" @@ -176,8 +177,8 @@ func setHostOptions(hostSubnetPrefix, ncSubnetPrefix *net.IPNet, options map[str }, } - azureDNSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncSubnetPrefix.String(), iptables.AzureDNS, iptables.UDP, iptables.DNSPort) - azureIMDSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncSubnetPrefix.String(), iptables.AzureIMDS, iptables.TCP, iptables.HTTPPort) + azureDNSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncSubnetPrefix.String(), networkutils.AzureDNS, iptables.UDP, iptables.DNSPort) + azureIMDSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncSubnetPrefix.String(), networkutils.AzureIMDS, iptables.TCP, iptables.HTTPPort) snatPrimaryIPJump := fmt.Sprintf("%s --to %s", iptables.Snat, info.ncPrimaryIP) // we need to snat IMDS traffic to node IP, this sets up snat '--to' diff --git a/cni/network/network.go b/cni/network/network.go index 4d621d0d32..fbfc186a70 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -14,6 +14,7 @@ import ( "github.com/Azure/azure-container-networking/aitelemetry" "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cni/api" + "github.com/Azure/azure-container-networking/cni/util" "github.com/Azure/azure-container-networking/cns" cnscli "github.com/Azure/azure-container-networking/cns/client" "github.com/Azure/azure-container-networking/common" @@ -63,13 +64,6 @@ const ( jsonFileExtension = ".json" ) -type ExecutionMode string - -const ( - Default ExecutionMode = "default" - Baremetal ExecutionMode = "baremetal" -) - // NetPlugin represents the CNI network plugin. type NetPlugin struct { *cni.Plugin @@ -402,7 +396,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { } log.Printf("Execution mode :%s", nwCfg.ExecutionMode) - if nwCfg.ExecutionMode == string(Baremetal) { + if nwCfg.ExecutionMode == string(util.Baremetal) { var res *nnscontracts.ConfigureContainerNetworkingResponse log.Printf("Baremetal mode. Calling vnet agent for ADD") res, err = plugin.nnsClient.AddContainerNetworking(context.Background(), k8sPodName, args.Netns) @@ -505,10 +499,6 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { // Allocate from azure ipam if !nwCfg.MultiTenancy { - if nwCfg.Ipam.Type == network.AzureCNS { - enableSnatForDns = true - } - result, resultV6, err = plugin.ipamInvoker.Add(nwCfg, args, &subnetPrefix, options) if err != nil { return err @@ -533,13 +523,25 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Created network %v with subnet %v.", networkID, subnetPrefix.String()) } - ncPrimaryIP, exists := options[network.SNATIPKey] - if !exists { - ncPrimaryIP = "" - } - - epInfo, err := plugin.createEndpointInternal(nwCfg, cnsNetworkConfig, result, resultV6, azIpamResult, args, &nwInfo, - policies, endpointId, k8sPodName, k8sNamespace, ncPrimaryIP.(string), enableInfraVnet, enableSnatForDns) + natInfo := getNATInfo(nwCfg.ExecutionMode, options[network.SNATIPKey], nwCfg.MultiTenancy, enableSnatForDns) + + createEndpointInternalOpt := createEndpointInternalOpt{ + nwCfg: nwCfg, + cnsNetworkConfig: cnsNetworkConfig, + result: result, + resultV6: resultV6, + azIpamResult: azIpamResult, + args: args, + nwInfo: &nwInfo, + policies: policies, + endpointID: endpointId, + k8sPodName: k8sPodName, + k8sNamespace: k8sNamespace, + enableInfraVnet: enableInfraVnet, + enableSnatForDNS: enableSnatForDns, + natInfo: natInfo, + } + epInfo, err := plugin.createEndpointInternal(&createEndpointInternalOpt) if err != nil { log.Errorf("Endpoint creation failed:%w", err) return err @@ -665,111 +667,112 @@ func (plugin *NetPlugin) createNetworkInternal( return nwInfo, err } -func (plugin *NetPlugin) createEndpointInternal( - nwCfg *cni.NetworkConfig, - cnsNetworkConfig *cns.GetNetworkContainerResponse, - result *cniTypesCurr.Result, - resultV6 *cniTypesCurr.Result, - azIpamResult *cniTypesCurr.Result, - args *cniSkel.CmdArgs, - nwInfo *network.NetworkInfo, - policies []policy.Policy, - endpointID string, - k8sPodName string, - k8sNamespace string, - ncPrimaryIP string, - enableInfraVnet bool, - enableSnatForDNS bool, -) (network.EndpointInfo, error) { +type createEndpointInternalOpt struct { + nwCfg *cni.NetworkConfig + cnsNetworkConfig *cns.GetNetworkContainerResponse + result *cniTypesCurr.Result + resultV6 *cniTypesCurr.Result + azIpamResult *cniTypesCurr.Result + args *cniSkel.CmdArgs + nwInfo *network.NetworkInfo + policies []policy.Policy + endpointID string + k8sPodName string + k8sNamespace string + enableInfraVnet bool + enableSnatForDNS bool + natInfo []policy.NATInfo +} + +func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) (network.EndpointInfo, error) { epInfo := network.EndpointInfo{} - epDNSInfo, err := getEndpointDNSSettings(nwCfg, result, k8sNamespace) + + epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, opt.result, opt.k8sNamespace) if err != nil { err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) return epInfo, err } - if nwCfg.IPV6Mode == network.IPV6Nat { + if opt.nwCfg.IPV6Mode == network.IPV6Nat { var ipv6Policy policy.Policy - ipv6Policy, err = addIPV6EndpointPolicy(*nwInfo) + ipv6Policy, err = addIPV6EndpointPolicy(*opt.nwInfo) if err != nil { err = plugin.Errorf("Failed to set ipv6 endpoint policy: %v", err) return epInfo, err } - policies = append(policies, ipv6Policy) + opt.policies = append(opt.policies, ipv6Policy) } - vethName := fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName) - if nwCfg.Mode != opModeTransparent { + vethName := fmt.Sprintf("%s.%s", opt.k8sNamespace, opt.k8sPodName) + if opt.nwCfg.Mode != opModeTransparent { // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. // IT will result in unpredictable behavior if API server decides to // reorder DELETE and ADD call for new incarnation of same POD. - vethName = fmt.Sprintf("%s%s%s", nwInfo.Id, args.ContainerID, args.IfName) + vethName = fmt.Sprintf("%s%s%s", opt.nwInfo.Id, opt.args.ContainerID, opt.args.IfName) } epInfo = network.EndpointInfo{ - Id: endpointID, - ContainerID: args.ContainerID, - NetNsPath: args.Netns, - IfName: args.IfName, + Id: opt.endpointID, + ContainerID: opt.args.ContainerID, + NetNsPath: opt.args.Netns, + IfName: opt.args.IfName, Data: make(map[string]interface{}), DNS: epDNSInfo, - Policies: policies, - IPsToRouteViaHost: nwCfg.IPsToRouteViaHost, - EnableSnatOnHost: nwCfg.EnableSnatOnHost, - EnableMultiTenancy: nwCfg.MultiTenancy, - EnableInfraVnet: enableInfraVnet, - EnableSnatForDns: enableSnatForDNS, - IsAksSwift: nwCfg.Ipam.Type == network.AzureCNS && !nwCfg.MultiTenancy, - PODName: k8sPodName, - PODNameSpace: k8sNamespace, + Policies: opt.policies, + IPsToRouteViaHost: opt.nwCfg.IPsToRouteViaHost, + EnableSnatOnHost: opt.nwCfg.EnableSnatOnHost, + EnableMultiTenancy: opt.nwCfg.MultiTenancy, + EnableInfraVnet: opt.enableInfraVnet, + EnableSnatForDns: opt.enableSnatForDNS, + PODName: opt.k8sPodName, + PODNameSpace: opt.k8sNamespace, SkipHotAttachEp: false, // Hot attach at the time of endpoint creation - IPV6Mode: nwCfg.IPV6Mode, - VnetCidrs: nwCfg.VnetCidrs, - ServiceCidrs: nwCfg.ServiceCidrs, + IPV6Mode: opt.nwCfg.IPV6Mode, + VnetCidrs: opt.nwCfg.VnetCidrs, + ServiceCidrs: opt.nwCfg.ServiceCidrs, + NATInfo: opt.natInfo, } - epPolicies := getPoliciesFromRuntimeCfg(nwCfg) + epPolicies := getPoliciesFromRuntimeCfg(opt.nwCfg) epInfo.Policies = append(epInfo.Policies, epPolicies...) - epInfo.Data[policy.NcPrimaryIPKey] = ncPrimaryIP - // Populate addresses. - for _, ipconfig := range result.IPs { + for _, ipconfig := range opt.result.IPs { epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) } - if resultV6 != nil { - for _, ipconfig := range resultV6.IPs { + if opt.resultV6 != nil { + for _, ipconfig := range opt.resultV6.IPs { epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address) } } // Populate routes. - for _, route := range result.Routes { + for _, route := range opt.result.Routes { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } - if azIpamResult != nil && azIpamResult.IPs != nil { - epInfo.InfraVnetIP = azIpamResult.IPs[0].Address + if opt.azIpamResult != nil && opt.azIpamResult.IPs != nil { + epInfo.InfraVnetIP = opt.azIpamResult.IPs[0].Address } - if nwCfg.MultiTenancy { - plugin.multitenancyClient.SetupRoutingForMultitenancy(nwCfg, cnsNetworkConfig, azIpamResult, &epInfo, result) + if opt.nwCfg.MultiTenancy { + plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, opt.result) } - setEndpointOptions(cnsNetworkConfig, &epInfo, vethName) + setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName) - cnsclient, err := cnscli.New(nwCfg.CNSUrl, defaultRequestTimeout) + cnsclient, err := cnscli.New(opt.nwCfg.CNSUrl, defaultRequestTimeout) if err != nil { - log.Printf("failed to initialized cns client with URL %s: %v", nwCfg.CNSUrl, err.Error()) + log.Printf("failed to initialized cns client with URL %s: %v", opt.nwCfg.CNSUrl, err.Error()) return epInfo, plugin.Errorf(err.Error()) } // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) - err = plugin.nm.CreateEndpoint(cnsclient, nwInfo.Id, &epInfo) + err = plugin.nm.CreateEndpoint(cnsclient, opt.nwInfo.Id, &epInfo) if err != nil { err = plugin.Errorf("Failed to create endpoint: %v", err) } @@ -925,7 +928,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { } log.Printf("Execution mode :%s", nwCfg.ExecutionMode) - if nwCfg.ExecutionMode == string(Baremetal) { + if nwCfg.ExecutionMode == string(util.Baremetal) { log.Printf("Baremetal mode. Calling vnet agent for delete container") diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 401c9cca6d..5cd627ceb0 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -135,3 +135,7 @@ func updateSubnetPrefix(cnsNetworkConfig *cns.GetNetworkContainerResponse, subne func (plugin *NetPlugin) getNetworkName(podName, podNs, ifName string, nwCfg *cni.NetworkConfig) (string, error) { return nwCfg.Name, nil } + +func getNATInfo(_ string, _ interface{}, _, _ bool) (natInfo []policy.NATInfo) { + return natInfo +} diff --git a/cni/network/network_test.go b/cni/network/network_test.go index ccccba4986..5e4cefacee 100644 --- a/cni/network/network_test.go +++ b/cni/network/network_test.go @@ -8,6 +8,7 @@ import ( "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cni/api" + "github.com/Azure/azure-container-networking/cni/util" "github.com/Azure/azure-container-networking/common" acnnetwork "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/nns" @@ -703,16 +704,15 @@ func TestPluginMultitenancyDelete(t *testing.T) { } /* -Baremetal scenarios + Baremetal scenarios */ - func TestPluginBaremetalAdd(t *testing.T) { plugin, _ := cni.NewPlugin("test", "0.3.0") localNwCfg := cni.NetworkConfig{ CNIVersion: "0.3.0", Name: "baremetal-net", - ExecutionMode: string(Baremetal), + ExecutionMode: string(util.Baremetal), EnableExactMatchForPodName: true, Master: "eth0", } @@ -783,7 +783,7 @@ func TestPluginBaremetalDelete(t *testing.T) { localNwCfg := cni.NetworkConfig{ CNIVersion: "0.3.0", Name: "baremetal-net", - ExecutionMode: string(Baremetal), + ExecutionMode: string(util.Baremetal), EnableExactMatchForPodName: true, Master: "eth0", } @@ -832,6 +832,111 @@ func TestPluginBaremetalDelete(t *testing.T) { } } +/* + AKS-Swift scenario +*/ +func TestPluginAKSSwiftAdd(t *testing.T) { + plugin := GetTestResources() + + localNwCfg := cni.NetworkConfig{ + CNIVersion: "0.3.0", + Name: "aksswift-net", + ExecutionMode: string(util.AKSSwift), + EnableExactMatchForPodName: true, + Master: "eth0", + } + + tests := []struct { + name string + plugin *NetPlugin + args *cniSkel.CmdArgs + wantErr bool + wantErrMsg string + }{ + { + name: "AKS Swift Add Happy path", + plugin: plugin, + args: &cniSkel.CmdArgs{ + StdinData: localNwCfg.Serialize(), + ContainerID: "test-container", + Netns: "test-container", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.plugin.Add(tt.args) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErrMsg, "Expected %v but got %+v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + endpoints, _ := plugin.nm.GetAllEndpoints(localNwCfg.Name) + require.Condition(t, assert.Comparison(func() bool { return len(endpoints) == 1 })) + } + }) + } +} + +func TestPluginAKSSwiftDelete(t *testing.T) { + plugin := GetTestResources() + localNwCfg := cni.NetworkConfig{ + CNIVersion: "0.3.0", + Name: "aksswift-net", + ExecutionMode: string(util.AKSSwift), + EnableExactMatchForPodName: true, + Master: "eth0", + } + + tests := []struct { + name string + methods []string + args *cniSkel.CmdArgs + wantErr bool + wantErrMsg string + }{ + { + name: "AKS Swift delete success", + methods: []string{CNI_ADD, CNI_DEL}, + args: &cniSkel.CmdArgs{ + StdinData: localNwCfg.Serialize(), + ContainerID: "test-container", + Netns: "test-container", + Args: fmt.Sprintf("K8S_POD_NAME=%v;K8S_POD_NAMESPACE=%v", "test-pod", "test-pod-ns"), + IfName: eth0IfName, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + var err error + for _, method := range tt.methods { + if method == CNI_ADD { + err = plugin.Add(tt.args) + } else if method == CNI_DEL { + err = plugin.Delete(tt.args) + } + } + + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + endpoints, _ := plugin.nm.GetAllEndpoints(localNwCfg.Name) + require.Condition(t, assert.Comparison(func() bool { return len(endpoints) == 0 })) + } + }) + } +} + func TestNewPlugin(t *testing.T) { tests := []struct { name string diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 81992da105..d4c8d58426 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -11,9 +11,11 @@ import ( "strings" "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cni/util" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" hnsv2 "github.com/Microsoft/hcsshim/hcn" @@ -348,3 +350,18 @@ func determineWinVer() { log.Errorf(err.Error()) } } + +func getNATInfo(executionMode string, ncPrimaryIPIface interface{}, multitenancy, enableSnatForDNS bool) (natInfo []policy.NATInfo) { + if executionMode == string(util.AKSSwift) { + ncPrimaryIP := "" + if ncPrimaryIPIface != nil { + ncPrimaryIP = ncPrimaryIPIface.(string) + } + + natInfo = append(natInfo, []policy.NATInfo{{VirtualIP: ncPrimaryIP, Destinations: []string{networkutils.AzureDNS}}, {Destinations: []string{networkutils.AzureIMDS}}}...) + } else if multitenancy && enableSnatForDNS { + natInfo = append(natInfo, policy.NATInfo{Destinations: []string{networkutils.AzureDNS}}) + } + + return natInfo +} diff --git a/cni/util/const.go b/cni/util/const.go new file mode 100644 index 0000000000..f49d18bd5a --- /dev/null +++ b/cni/util/const.go @@ -0,0 +1,10 @@ +package util + +type ExecutionMode string + +// CNI execution modes +const ( + Default ExecutionMode = "default" + Baremetal ExecutionMode = "baremetal" + AKSSwift ExecutionMode = "aksswift" +) diff --git a/iptables/iptables.go b/iptables/iptables.go index 79d1b28fdf..bc8feb9aca 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -77,12 +77,6 @@ const ( TCP = "tcp" ) -// known IP's -const ( - AzureDNS = "168.63.129.16" - AzureIMDS = "169.254.169.254" -) - var DisableIPTableLock bool type IPTableEntry struct { diff --git a/network/endpoint.go b/network/endpoint.go index ee09833c39..ce295aff86 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -69,7 +69,6 @@ type EndpointInfo struct { EnableInfraVnet bool EnableMultiTenancy bool EnableSnatForDns bool - IsAksSwift bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool NetworkContainerID string @@ -81,6 +80,7 @@ type EndpointInfo struct { IPV6Mode string VnetCidrs string ServiceCidrs string + NATInfo []policy.NATInfo } // RouteInfo contains information about an IP route. diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 7050ad3245..01322a08c8 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -249,8 +249,7 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE MacAddress: epInfo.MacAddress.String(), } - if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data, - epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy, epInfo.IsAksSwift); err == nil { + if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy, epInfo.NATInfo); err == nil { for _, epPolicy := range endpointPolicies { hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) } diff --git a/network/networkutils/networkutils.go b/network/networkutils/networkutils.go new file mode 100644 index 0000000000..68b030fd97 --- /dev/null +++ b/network/networkutils/networkutils.go @@ -0,0 +1,7 @@ +package networkutils + +// known IP's +const ( + AzureDNS = "168.63.129.16" + AzureIMDS = "169.254.169.254" +) diff --git a/network/policy/policy.go b/network/policy/policy.go index e1a4fc3e45..677decadb5 100644 --- a/network/policy/policy.go +++ b/network/policy/policy.go @@ -4,11 +4,6 @@ import ( "encoding/json" ) -const ( - // NcPrimaryIPKey indicates constant for the key string - NcPrimaryIPKey string = "NCPrimaryIPKey" -) - const ( NetworkPolicy CNIPolicyType = "NetworkPolicy" EndpointPolicy CNIPolicyType = "EndpointPolicy" @@ -25,3 +20,9 @@ type Policy struct { Type CNIPolicyType Data json.RawMessage } + +// NATInfo contains information about NAT rules +type NATInfo struct { + Destinations []string + VirtualIP string +} diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go index b7482a41af..68abded660 100644 --- a/network/policy/policy_windows.go +++ b/network/policy/policy_windows.go @@ -4,8 +4,8 @@ import ( "encoding/json" "fmt" - "github.com/Azure/azure-container-networking/iptables" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" ) @@ -428,11 +428,9 @@ func GetHcnL4WFPProxyPolicy(policy Policy) (hcn.EndpointPolicy, error) { } // GetHcnEndpointPolicies returns array of all endpoint policies. -func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}, enableSnatForDns, enableMultiTenancy, isAksSwift bool) ([]hcn.EndpointPolicy, error) { - var ( - hcnEndPointPolicies []hcn.EndpointPolicy - snatAndSerialize = (enableMultiTenancy && enableSnatForDns) || isAksSwift - ) +func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}, enableSnatForDns, enableMultiTenancy bool, natInfo []NATInfo) ([]hcn.EndpointPolicy, error) { + var hcnEndPointPolicies []hcn.EndpointPolicy + for _, policy := range policies { if policy.Type == policyType { var err error @@ -468,27 +466,17 @@ func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoD } } - if snatAndSerialize && ValidWinVerForDnsNat { - var snatToIP string - if isAksSwift { - snatToIP = epInfoData[NcPrimaryIPKey].(string) - - imdsNatPolicy, err := AddImdsNATPolicyV2() + if ValidWinVerForDnsNat { + for _, natRule := range natInfo { + natPolicy, err := AddNATPolicyV2(natRule.VirtualIP, natRule.Destinations) if err != nil { - log.Printf("Failed to retrieve ImdsNAT endpoint policy due to error: %v", err) + log.Printf("Failed to retrieve NAT endpoint policy due to error: %v", err) return hcnEndPointPolicies, err } - hcnEndPointPolicies = append(hcnEndPointPolicies, imdsNatPolicy) + hcnEndPointPolicies = append(hcnEndPointPolicies, natPolicy) + log.Printf("Successfully set the policy: %+v", natPolicy) } - - dnsNatPolicy, err := AddDnsNATPolicyV2(snatToIP) - if err != nil { - log.Printf("Failed to retrieve DnsNAT endpoint policy due to error: %v", err) - return hcnEndPointPolicies, err - } - - hcnEndPointPolicies = append(hcnEndPointPolicies, dnsNatPolicy) } return hcnEndPointPolicies, nil @@ -498,26 +486,15 @@ func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoD func AddDnsNATPolicyV1() (json.RawMessage, error) { outBoundNatPolicy := hcsshim.OutboundNatPolicy{ Policy: hcsshim.Policy{Type: hcsshim.OutboundNat}, - Destinations: []string{iptables.AzureDNS}, + Destinations: []string{networkutils.AzureDNS}, } serializedPolicy, err := json.Marshal(outBoundNatPolicy) return serializedPolicy, err } -// AddDnsNATPolicyV2 returns DNS NAT endpoint policy for HNSv2 -func AddDnsNATPolicyV2(snatToIP string) (hcn.EndpointPolicy, error) { - outBoundNatPolicySettings := hcn.OutboundNatPolicySetting{VirtualIP: snatToIP, Destinations: []string{iptables.AzureDNS}} - outBoundNatPolicySettingsBytes, err := json.Marshal(outBoundNatPolicySettings) - endpointPolicy := hcn.EndpointPolicy{ - Type: hcn.OutBoundNAT, - Settings: outBoundNatPolicySettingsBytes, - } - return endpointPolicy, err -} - -// AddImdsNATPolicyV2 returns IMDS NAT endpoint policy for HNSv2 -func AddImdsNATPolicyV2() (hcn.EndpointPolicy, error) { - outBoundNatPolicySettings := hcn.OutboundNatPolicySetting{Destinations: []string{iptables.AzureIMDS}} +// AddNATPolicyV2 returns serialized endpoint policy based on vip (IP to snat to) and destination(s) +func AddNATPolicyV2(vip string, destinations []string) (hcn.EndpointPolicy, error) { + outBoundNatPolicySettings := hcn.OutboundNatPolicySetting{VirtualIP: vip, Destinations: destinations} outBoundNatPolicySettingsBytes, err := json.Marshal(outBoundNatPolicySettings) endpointPolicy := hcn.EndpointPolicy{ Type: hcn.OutBoundNAT,