Skip to content

fix: Add NAT Policies for Windows AKS-Swift Scenario #1036

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 20, 2021
Merged

fix: Add NAT Policies for Windows AKS-Swift Scenario #1036

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9dd980f
Add DNS NAT policy for windows AKS-Swift scenario
Sep 28, 2021
e135969
Addressing comments
Oct 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cni/network/invoker_cns.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/Azure/azure-container-networking/iptables"
"github.com/Azure/azure-container-networking/log"
"github.com/Azure/azure-container-networking/network"
"github.com/Azure/azure-container-networking/network/networkutils"
cniSkel "github.com/containernetworking/cni/pkg/skel"
cniTypes "github.com/containernetworking/cni/pkg/types"
cniTypesCurr "github.com/containernetworking/cni/pkg/types/current"
Expand Down Expand Up @@ -176,8 +177,8 @@ func setHostOptions(hostSubnetPrefix, ncSubnetPrefix *net.IPNet, options map[str
},
}

azureDNSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncSubnetPrefix.String(), iptables.AzureDNS, iptables.UDP, iptables.DNSPort)
azureIMDSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncSubnetPrefix.String(), iptables.AzureIMDS, iptables.TCP, iptables.HTTPPort)
azureDNSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncSubnetPrefix.String(), networkutils.AzureDNS, iptables.UDP, iptables.DNSPort)
azureIMDSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncSubnetPrefix.String(), networkutils.AzureIMDS, iptables.TCP, iptables.HTTPPort)

snatPrimaryIPJump := fmt.Sprintf("%s --to %s", iptables.Snat, info.ncPrimaryIP)
// we need to snat IMDS traffic to node IP, this sets up snat '--to'
Expand Down
139 changes: 77 additions & 62 deletions cni/network/network.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/Azure/azure-container-networking/aitelemetry"
"github.com/Azure/azure-container-networking/cni"
"github.com/Azure/azure-container-networking/cni/api"
"github.com/Azure/azure-container-networking/cni/util"
"github.com/Azure/azure-container-networking/cns"
cnscli "github.com/Azure/azure-container-networking/cns/client"
"github.com/Azure/azure-container-networking/common"
Expand Down Expand Up @@ -63,13 +64,6 @@ const (
jsonFileExtension = ".json"
)

type ExecutionMode string

const (
Default ExecutionMode = "default"
Baremetal ExecutionMode = "baremetal"
)

// NetPlugin represents the CNI network plugin.
type NetPlugin struct {
*cni.Plugin
Expand Down Expand Up @@ -402,7 +396,7 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error {
}

log.Printf("Execution mode :%s", nwCfg.ExecutionMode)
if nwCfg.ExecutionMode == string(Baremetal) {
if nwCfg.ExecutionMode == string(util.Baremetal) {
var res *nnscontracts.ConfigureContainerNetworkingResponse
log.Printf("Baremetal mode. Calling vnet agent for ADD")
res, err = plugin.nnsClient.AddContainerNetworking(context.Background(), k8sPodName, args.Netns)
Expand Down Expand Up @@ -529,8 +523,25 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error {
log.Printf("[cni-net] Created network %v with subnet %v.", networkID, subnetPrefix.String())
}

epInfo, err := plugin.createEndpointInternal(nwCfg, cnsNetworkConfig, result, resultV6, azIpamResult, args, &nwInfo,
policies, endpointId, k8sPodName, k8sNamespace, enableInfraVnet, enableSnatForDns)
natInfo := getNATInfo(nwCfg.ExecutionMode, options[network.SNATIPKey], nwCfg.MultiTenancy, enableSnatForDns)

createEndpointInternalOpt := createEndpointInternalOpt{
nwCfg: nwCfg,
cnsNetworkConfig: cnsNetworkConfig,
result: result,
resultV6: resultV6,
azIpamResult: azIpamResult,
args: args,
nwInfo: &nwInfo,
policies: policies,
endpointID: endpointId,
k8sPodName: k8sPodName,
k8sNamespace: k8sNamespace,
enableInfraVnet: enableInfraVnet,
enableSnatForDNS: enableSnatForDns,
natInfo: natInfo,
}
epInfo, err := plugin.createEndpointInternal(&createEndpointInternalOpt)
if err != nil {
log.Errorf("Endpoint creation failed:%w", err)
return err
Expand Down Expand Up @@ -656,108 +667,112 @@ func (plugin *NetPlugin) createNetworkInternal(
return nwInfo, err
}

func (plugin *NetPlugin) createEndpointInternal(
nwCfg *cni.NetworkConfig,
cnsNetworkConfig *cns.GetNetworkContainerResponse,
result *cniTypesCurr.Result,
resultV6 *cniTypesCurr.Result,
azIpamResult *cniTypesCurr.Result,
args *cniSkel.CmdArgs,
nwInfo *network.NetworkInfo,
policies []policy.Policy,
endpointID string,
k8sPodName string,
k8sNamespace string,
enableInfraVnet bool,
enableSnatForDNS bool,
) (network.EndpointInfo, error) {
type createEndpointInternalOpt struct {
nwCfg *cni.NetworkConfig
cnsNetworkConfig *cns.GetNetworkContainerResponse
result *cniTypesCurr.Result
resultV6 *cniTypesCurr.Result
azIpamResult *cniTypesCurr.Result
args *cniSkel.CmdArgs
nwInfo *network.NetworkInfo
policies []policy.Policy
endpointID string
k8sPodName string
k8sNamespace string
enableInfraVnet bool
enableSnatForDNS bool
natInfo []policy.NATInfo
}

func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) (network.EndpointInfo, error) {
epInfo := network.EndpointInfo{}
epDNSInfo, err := getEndpointDNSSettings(nwCfg, result, k8sNamespace)

epDNSInfo, err := getEndpointDNSSettings(opt.nwCfg, opt.result, opt.k8sNamespace)
if err != nil {
err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err)
return epInfo, err
}

if nwCfg.IPV6Mode == network.IPV6Nat {
if opt.nwCfg.IPV6Mode == network.IPV6Nat {
var ipv6Policy policy.Policy

ipv6Policy, err = addIPV6EndpointPolicy(*nwInfo)
ipv6Policy, err = addIPV6EndpointPolicy(*opt.nwInfo)
if err != nil {
err = plugin.Errorf("Failed to set ipv6 endpoint policy: %v", err)
return epInfo, err
}

policies = append(policies, ipv6Policy)
opt.policies = append(opt.policies, ipv6Policy)
}

vethName := fmt.Sprintf("%s.%s", k8sNamespace, k8sPodName)
if nwCfg.Mode != opModeTransparent {
vethName := fmt.Sprintf("%s.%s", opt.k8sNamespace, opt.k8sPodName)
if opt.nwCfg.Mode != opModeTransparent {
// this mechanism of using only namespace and name is not unique for different incarnations of POD/container.
// IT will result in unpredictable behavior if API server decides to
// reorder DELETE and ADD call for new incarnation of same POD.
vethName = fmt.Sprintf("%s%s%s", nwInfo.Id, args.ContainerID, args.IfName)
vethName = fmt.Sprintf("%s%s%s", opt.nwInfo.Id, opt.args.ContainerID, opt.args.IfName)
}

epInfo = network.EndpointInfo{
Id: endpointID,
ContainerID: args.ContainerID,
NetNsPath: args.Netns,
IfName: args.IfName,
Id: opt.endpointID,
ContainerID: opt.args.ContainerID,
NetNsPath: opt.args.Netns,
IfName: opt.args.IfName,
Data: make(map[string]interface{}),
DNS: epDNSInfo,
Policies: policies,
IPsToRouteViaHost: nwCfg.IPsToRouteViaHost,
EnableSnatOnHost: nwCfg.EnableSnatOnHost,
EnableMultiTenancy: nwCfg.MultiTenancy,
EnableInfraVnet: enableInfraVnet,
EnableSnatForDns: enableSnatForDNS,
PODName: k8sPodName,
PODNameSpace: k8sNamespace,
Policies: opt.policies,
IPsToRouteViaHost: opt.nwCfg.IPsToRouteViaHost,
EnableSnatOnHost: opt.nwCfg.EnableSnatOnHost,
EnableMultiTenancy: opt.nwCfg.MultiTenancy,
EnableInfraVnet: opt.enableInfraVnet,
EnableSnatForDns: opt.enableSnatForDNS,
PODName: opt.k8sPodName,
PODNameSpace: opt.k8sNamespace,
SkipHotAttachEp: false, // Hot attach at the time of endpoint creation
IPV6Mode: nwCfg.IPV6Mode,
VnetCidrs: nwCfg.VnetCidrs,
ServiceCidrs: nwCfg.ServiceCidrs,
IPV6Mode: opt.nwCfg.IPV6Mode,
VnetCidrs: opt.nwCfg.VnetCidrs,
ServiceCidrs: opt.nwCfg.ServiceCidrs,
NATInfo: opt.natInfo,
}

epPolicies := getPoliciesFromRuntimeCfg(nwCfg)

epPolicies := getPoliciesFromRuntimeCfg(opt.nwCfg)
epInfo.Policies = append(epInfo.Policies, epPolicies...)

// Populate addresses.
for _, ipconfig := range result.IPs {
for _, ipconfig := range opt.result.IPs {
epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address)
}

if resultV6 != nil {
for _, ipconfig := range resultV6.IPs {
if opt.resultV6 != nil {
for _, ipconfig := range opt.resultV6.IPs {
epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address)
}
}

// Populate routes.
for _, route := range result.Routes {
for _, route := range opt.result.Routes {
epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW})
}

if azIpamResult != nil && azIpamResult.IPs != nil {
epInfo.InfraVnetIP = azIpamResult.IPs[0].Address
if opt.azIpamResult != nil && opt.azIpamResult.IPs != nil {
epInfo.InfraVnetIP = opt.azIpamResult.IPs[0].Address
}

if nwCfg.MultiTenancy {
plugin.multitenancyClient.SetupRoutingForMultitenancy(nwCfg, cnsNetworkConfig, azIpamResult, &epInfo, result)
if opt.nwCfg.MultiTenancy {
plugin.multitenancyClient.SetupRoutingForMultitenancy(opt.nwCfg, opt.cnsNetworkConfig, opt.azIpamResult, &epInfo, opt.result)
}

setEndpointOptions(cnsNetworkConfig, &epInfo, vethName)
setEndpointOptions(opt.cnsNetworkConfig, &epInfo, vethName)

cnsclient, err := cnscli.New(nwCfg.CNSUrl, defaultRequestTimeout)
cnsclient, err := cnscli.New(opt.nwCfg.CNSUrl, defaultRequestTimeout)
if err != nil {
log.Printf("failed to initialized cns client with URL %s: %v", nwCfg.CNSUrl, err.Error())
log.Printf("failed to initialized cns client with URL %s: %v", opt.nwCfg.CNSUrl, err.Error())
return epInfo, plugin.Errorf(err.Error())
}

// Create the endpoint.
log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id)
err = plugin.nm.CreateEndpoint(cnsclient, nwInfo.Id, &epInfo)
err = plugin.nm.CreateEndpoint(cnsclient, opt.nwInfo.Id, &epInfo)
if err != nil {
err = plugin.Errorf("Failed to create endpoint: %v", err)
}
Expand Down Expand Up @@ -913,7 +928,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error {
}

log.Printf("Execution mode :%s", nwCfg.ExecutionMode)
if nwCfg.ExecutionMode == string(Baremetal) {
if nwCfg.ExecutionMode == string(util.Baremetal) {

log.Printf("Baremetal mode. Calling vnet agent for delete container")

Expand Down
4 changes: 4 additions & 0 deletions cni/network/network_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,3 +135,7 @@ func updateSubnetPrefix(cnsNetworkConfig *cns.GetNetworkContainerResponse, subne
func (plugin *NetPlugin) getNetworkName(podName, podNs, ifName string, nwCfg *cni.NetworkConfig) (string, error) {
return nwCfg.Name, nil
}

func getNATInfo(_ string, _ interface{}, _, _ bool) (natInfo []policy.NATInfo) {
return natInfo
}
Loading