From 4c47d59606bfca3e32ad808c7b8f93aabd5f8213 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 18 Nov 2021 13:34:00 -0800 Subject: [PATCH 1/5] added dsr changes for windows --- cni/azure-windows-baremetal.conflist | 3 ++ cni/azure-windows-multitenancy.conflist | 3 ++ cni/netconfig.go | 12 ++++--- cni/network/network.go | 15 ++++----- cni/network/network_windows.go | 43 +++++++++++++++++++++++-- network/policy/policy.go | 6 ++++ network/policy/policy_windows.go | 43 +++++++++++++++++++++++++ 7 files changed, 109 insertions(+), 16 deletions(-) diff --git a/cni/azure-windows-baremetal.conflist b/cni/azure-windows-baremetal.conflist index 3916597e92..be8cf2b932 100644 --- a/cni/azure-windows-baremetal.conflist +++ b/cni/azure-windows-baremetal.conflist @@ -25,6 +25,9 @@ "svc.cluster.local" ] }, + windowsSettings: { + "disableLoopbackDSR": true + }, "AdditionalArgs": [ { "Name": "EndpointPolicy", diff --git a/cni/azure-windows-multitenancy.conflist b/cni/azure-windows-multitenancy.conflist index 38833b8320..d80ae3181b 100644 --- a/cni/azure-windows-multitenancy.conflist +++ b/cni/azure-windows-multitenancy.conflist @@ -24,6 +24,9 @@ "svc.cluster.local" ] }, + windowsSettings: { + "disableLoopbackDSR": true + }, "AdditionalArgs": [ { "Name": "EndpointPolicy", diff --git a/cni/netconfig.go b/cni/netconfig.go index cde3d8accb..b95a7a6a79 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/Azure/azure-container-networking/network/policy" - cniTypes "github.com/containernetworking/cni/pkg/types" ) @@ -73,9 +72,14 @@ type NetworkConfig struct { Address string `json:"ipAddress,omitempty"` QueryInterval string `json:"queryInterval,omitempty"` } `json:"ipam,omitempty"` - DNS cniTypes.DNS `json:"dns,omitempty"` - RuntimeConfig RuntimeConfig `json:"runtimeConfig,omitempty"` - AdditionalArgs []KVPair `json:"AdditionalArgs,omitempty"` + DNS cniTypes.DNS `json:"dns,omitempty"` + RuntimeConfig RuntimeConfig `json:"runtimeConfig,omitempty"` + WindowsSettings WindowsSettings `json:"windowsSettings,omitempty"` + AdditionalArgs []KVPair `json:"AdditionalArgs,omitempty"` +} + +type WindowsSettings struct { + DisableLoopbackDSR bool `json:"disableLoopbackDSR,omitempty"` } type K8SPodEnvArgs struct { diff --git a/cni/network/network.go b/cni/network/network.go index fbfc186a70..3f89f9dde1 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -693,17 +693,14 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) return epInfo, err } - if opt.nwCfg.IPV6Mode == network.IPV6Nat { - var ipv6Policy policy.Policy + endpointPolicies, err := getEndpointPolicies(opt.nwCfg, opt.nwInfo, opt.result.IPs) + if err != nil { + log.Errorf("Failed to get endpoint policies:%v", err) + return epInfo, err + } - ipv6Policy, err = addIPV6EndpointPolicy(*opt.nwInfo) - if err != nil { - err = plugin.Errorf("Failed to set ipv6 endpoint policy: %v", err) - return epInfo, err - } + opt.policies = append(opt.policies, endpointPolicies...) - opt.policies = append(opt.policies, ipv6Policy) - } vethName := fmt.Sprintf("%s.%s", opt.k8sNamespace, opt.k8sPodName) if opt.nwCfg.Mode != opModeTransparent { diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index d4c8d58426..49bc36707f 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -19,11 +19,11 @@ import ( "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" hnsv2 "github.com/Microsoft/hcsshim/hcn" - "golang.org/x/sys/windows/registry" - cniSkel "github.com/containernetworking/cni/pkg/skel" cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" + "github.com/pkg/errors" + "golang.org/x/sys/windows/registry" ) var ( @@ -296,7 +296,44 @@ func getPoliciesFromRuntimeCfg(nwCfg *cni.NetworkConfig) []policy.Policy { return policies } -func addIPV6EndpointPolicy(nwInfo network.NetworkInfo) (policy.Policy, error) { +func getEndpointPolicies(nwCfg *cni.NetworkConfig, nwInfo *network.NetworkInfo, ipconfigs []*cniTypesCurr.IPConfig) ([]policy.Policy, error) { + var policies []policy.Policy + + if nwCfg.IPV6Mode == network.IPV6Nat { + ipv6Policy, err := addIPV6EndpointPolicy(nwInfo) + if err != nil { + return nil, errors.Wrap(err, "failed to set ipv6 endpoint policy") + } + policies = append(policies, ipv6Policy) + } + + if !nwCfg.WindowsSettings.DisableLoopbackDSR { + for _, config := range ipconfigs { + // consider DSR policy only for ipv4 address. Add for ipv6 when required + if config.Address.IP.To4() != nil { + dsrData := policy.LoopbackDSR{ + Type: policy.LoopbackDSRPolicy, + IPAddress: config.Address.IP, + } + + dsrDataBytes, err := json.Marshal(dsrData) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal dsr data") + } + dsrPolicy := policy.Policy{ + Type: policy.EndpointPolicy, + Data: dsrDataBytes, + } + policies = append(policies, dsrPolicy) + } + } + + } + + return policies, nil +} + +func addIPV6EndpointPolicy(nwInfo *network.NetworkInfo) (policy.Policy, error) { var eppolicy policy.Policy if len(nwInfo.Subnets) < 2 { diff --git a/network/policy/policy.go b/network/policy/policy.go index 677decadb5..6d5cbb89f0 100644 --- a/network/policy/policy.go +++ b/network/policy/policy.go @@ -2,6 +2,8 @@ package policy import ( "encoding/json" + + "github.com/pkg/errors" ) const ( @@ -12,6 +14,7 @@ const ( PortMappingPolicy CNIPolicyType = "NAT" ACLPolicy CNIPolicyType = "ACL" L4WFPProxyPolicy CNIPolicyType = "L4WFPPROXY" + LoopbackDSRPolicy CNIPolicyType = "LoopbackDSR" ) type CNIPolicyType string @@ -26,3 +29,6 @@ type NATInfo struct { Destinations []string VirtualIP string } + +// ErrIpv4NotFound ipv4 address not found in endpoint info +var ErrIpv4NotFound = errors.New("ipv4 not found in endpoint info") diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go index 68abded660..6cace557e9 100644 --- a/network/policy/policy_windows.go +++ b/network/policy/policy_windows.go @@ -3,11 +3,13 @@ package policy import ( "encoding/json" "fmt" + "net" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/networkutils" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" + "github.com/pkg/errors" ) const ( @@ -48,6 +50,11 @@ type KVPairL4WfpProxyPolicy struct { OutboundExceptions json.RawMessage `json:"OutboundExceptions"` } +type LoopbackDSR struct { + Type CNIPolicyType `json:"Type"` + IPAddress net.IP `json:"IPAddress"` +} + var ValidWinVerForDnsNat bool // SerializePolicies serializes policies to json. @@ -241,6 +248,13 @@ func GetPolicyType(policy Policy) CNIPolicyType { } } + // Check if the type is LoopbackDSRPolicy + var loopbackDsr LoopbackDSR + if err := json.Unmarshal(policy.Data, &loopbackDsr); err == nil { + if loopbackDsr.Type == LoopbackDSRPolicy { + return LoopbackDSRPolicy + } + } // Return empty string if the policy type is invalid log.Printf("Returning policyType INVALID") return "" @@ -427,6 +441,33 @@ func GetHcnL4WFPProxyPolicy(policy Policy) (hcn.EndpointPolicy, error) { return l4WfpEndpolicySetting, nil } +// GetHcnLoopbackDSRPolicy policy is for pod to reach itself by cluster service IP. +func GetHcnLoopbackDSRPolicy(policy Policy) (hcn.EndpointPolicy, error) { + var dsrData LoopbackDSR + + if err := hcn.DSRSupported(); err != nil { + return hcn.EndpointPolicy{}, errors.Wrap(err, "hcn doesn't support DSR") + } + + err := json.Unmarshal(policy.Data, &dsrData) + if err != nil { + return hcn.EndpointPolicy{}, errors.Wrap(err, "unmarshal dsr data failed") + } + + log.Printf("DSR policy for ip:%s", dsrData.IPAddress.String()) + + hcnLoopbackRoute := hcn.OutboundNatPolicySetting{ + Destinations: []string{dsrData.IPAddress.String()}, + } + rawPolicy, _ := json.Marshal(hcnLoopbackRoute) + endpointPolicy := hcn.EndpointPolicy{ + Type: hcn.OutBoundNAT, + Settings: rawPolicy, + } + + return endpointPolicy, nil +} + // GetHcnEndpointPolicies returns array of all endpoint policies. func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}, enableSnatForDns, enableMultiTenancy bool, natInfo []NATInfo) ([]hcn.EndpointPolicy, error) { var hcnEndPointPolicies []hcn.EndpointPolicy @@ -449,6 +490,8 @@ func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoD endpointPolicy, err = GetHcnACLPolicy(policy) case L4WFPProxyPolicy: endpointPolicy, err = GetHcnL4WFPProxyPolicy(policy) + case LoopbackDSRPolicy: + endpointPolicy, err = GetHcnLoopbackDSRPolicy(policy) default: // return error as we should be able to parse all the policies specified return hcnEndPointPolicies, fmt.Errorf("Failed to set Policy: Type: %s, Data: %s", policy.Type, policy.Data) From f86a0a2baa035da7ca9a0923f7911ced7c800a6b Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 18 Nov 2021 14:35:49 -0800 Subject: [PATCH 2/5] fixed lint and added unit test removed unused error --- cni/network/network_linux.go | 5 ++- cni/network/network_windows_test.go | 60 +++++++++++++++++++++++++++-- network/policy/policy.go | 5 --- 3 files changed, 61 insertions(+), 9 deletions(-) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 5cd627ceb0..b66786de39 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -9,7 +9,6 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/network/policy" - cniSkel "github.com/containernetworking/cni/pkg/skel" cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" @@ -118,6 +117,10 @@ func getEndpointDNSSettings(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Resul return getNetworkDNSSettings(nwCfg, result) } +func getEndpointPolicies(nwCfg *cni.NetworkConfig, nwInfo *network.NetworkInfo, ipconfigs []*cniTypesCurr.IPConfig) ([]policy.Policy, error) { + return nil, nil +} + // getPoliciesFromRuntimeCfg returns network policies from network config. // getPoliciesFromRuntimeCfg is a dummy function for Linux platform. func getPoliciesFromRuntimeCfg(nwCfg *cni.NetworkConfig) []policy.Policy { diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index 5e00390b67..7f5a506426 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -1,18 +1,20 @@ -//+build windows +//go:build windows +// +build windows package network import ( "fmt" - "github.com/Azure/azure-container-networking/cns" "net" "testing" "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/network/policy" "github.com/Azure/azure-container-networking/telemetry" "github.com/containernetworking/cni/pkg/skel" + cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -49,7 +51,7 @@ func TestAddWithRunTimeNetPolicies(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - p, err := addIPV6EndpointPolicy(tt.nwInfo) + p, err := addIPV6EndpointPolicy(&tt.nwInfo) if tt.wantErr { require.Error(t, err) } else { @@ -241,3 +243,55 @@ func TestSetPoliciesFromNwCfg(t *testing.T) { }) } } + +func TestDSRPolciy(t *testing.T) { + tests := []struct { + name string + nwCfg cni.NetworkConfig + nwInfo network.NetworkInfo + ipconfig []*cniTypesCurr.IPConfig + wantCount int + }{ + { + name: "test enable dsr policy", + nwCfg: cni.NetworkConfig{}, + nwInfo: network.NetworkInfo{}, + ipconfig: []*cniTypesCurr.IPConfig{ + { + Address: func() net.IPNet { + _, ipnet, _ := net.ParseCIDR("10.0.0.5/24") + return *ipnet + }(), + }, + }, + wantCount: 1, + }, + { + name: "test disable dsr policy", + nwCfg: cni.NetworkConfig{ + WindowsSettings: cni.WindowsSettings{ + DisableLoopbackDSR: true, + }, + }, + nwInfo: network.NetworkInfo{}, + ipconfig: []*cniTypesCurr.IPConfig{ + { + Address: func() net.IPNet { + _, ipnet, _ := net.ParseCIDR("10.0.0.5/24") + return *ipnet + }(), + }, + }, + wantCount: 0, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + policies, err := getEndpointPolicies(&tt.nwCfg, &tt.nwInfo, tt.ipconfig) + require.NoError(t, err) + require.Equal(t, tt.wantCount, len(policies)) + }) + } +} diff --git a/network/policy/policy.go b/network/policy/policy.go index 6d5cbb89f0..9cfd070356 100644 --- a/network/policy/policy.go +++ b/network/policy/policy.go @@ -2,8 +2,6 @@ package policy import ( "encoding/json" - - "github.com/pkg/errors" ) const ( @@ -29,6 +27,3 @@ type NATInfo struct { Destinations []string VirtualIP string } - -// ErrIpv4NotFound ipv4 address not found in endpoint info -var ErrIpv4NotFound = errors.New("ipv4 not found in endpoint info") From e321089326a897228767553dc97017eb024d0b71 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 19 Nov 2021 11:46:35 -0800 Subject: [PATCH 3/5] skip adding dsr policy for hnsv1 --- cni/network/network.go | 3 +-- cni/network/network_linux.go | 2 +- cni/network/network_windows.go | 6 +++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 3f89f9dde1..45db88c6b1 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -693,7 +693,7 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) return epInfo, err } - endpointPolicies, err := getEndpointPolicies(opt.nwCfg, opt.nwInfo, opt.result.IPs) + endpointPolicies, err := getEndpointPolicies(opt.nwCfg, opt.args.Netns, opt.nwInfo, opt.result.IPs) if err != nil { log.Errorf("Failed to get endpoint policies:%v", err) return epInfo, err @@ -701,7 +701,6 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) opt.policies = append(opt.policies, endpointPolicies...) - vethName := fmt.Sprintf("%s.%s", opt.k8sNamespace, opt.k8sPodName) if opt.nwCfg.Mode != opModeTransparent { // this mechanism of using only namespace and name is not unique for different incarnations of POD/container. diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index b66786de39..e9f68b84ac 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -117,7 +117,7 @@ func getEndpointDNSSettings(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Resul return getNetworkDNSSettings(nwCfg, result) } -func getEndpointPolicies(nwCfg *cni.NetworkConfig, nwInfo *network.NetworkInfo, ipconfigs []*cniTypesCurr.IPConfig) ([]policy.Policy, error) { +func getEndpointPolicies(nwCfg *cni.NetworkConfig, netns string, nwInfo *network.NetworkInfo, ipconfigs []*cniTypesCurr.IPConfig) ([]policy.Policy, error) { return nil, nil } diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 49bc36707f..150b134747 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -296,7 +296,7 @@ func getPoliciesFromRuntimeCfg(nwCfg *cni.NetworkConfig) []policy.Policy { return policies } -func getEndpointPolicies(nwCfg *cni.NetworkConfig, nwInfo *network.NetworkInfo, ipconfigs []*cniTypesCurr.IPConfig) ([]policy.Policy, error) { +func getEndpointPolicies(nwCfg *cni.NetworkConfig, netns string, nwInfo *network.NetworkInfo, ipconfigs []*cniTypesCurr.IPConfig) ([]policy.Policy, error) { var policies []policy.Policy if nwCfg.IPV6Mode == network.IPV6Nat { @@ -308,6 +308,10 @@ func getEndpointPolicies(nwCfg *cni.NetworkConfig, nwInfo *network.NetworkInfo, } if !nwCfg.WindowsSettings.DisableLoopbackDSR { + if isv2, _ := network.UseHnsV2(netns); !isv2 { + log.Printf("hnsv2 not supported. so skip adding dsr policy") + return nil, nil + } for _, config := range ipconfigs { // consider DSR policy only for ipv4 address. Add for ipv6 when required if config.Address.IP.To4() != nil { From c40bf3ad42e9da232f973ec67b9a914000ff6f5b Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 24 Nov 2021 15:47:24 -0800 Subject: [PATCH 4/5] addressed comments lint fix --- cni/azure-windows-baremetal.conflist | 3 - cni/azure-windows-multitenancy.conflist | 3 - cni/netconfig.go | 2 +- cni/network/network.go | 14 ++++- cni/network/network_linux.go | 2 +- cni/network/network_windows.go | 60 +++++++++++--------- network/policy/policy_windows.go | 74 +++++++++++++------------ 7 files changed, 85 insertions(+), 73 deletions(-) diff --git a/cni/azure-windows-baremetal.conflist b/cni/azure-windows-baremetal.conflist index be8cf2b932..3916597e92 100644 --- a/cni/azure-windows-baremetal.conflist +++ b/cni/azure-windows-baremetal.conflist @@ -25,9 +25,6 @@ "svc.cluster.local" ] }, - windowsSettings: { - "disableLoopbackDSR": true - }, "AdditionalArgs": [ { "Name": "EndpointPolicy", diff --git a/cni/azure-windows-multitenancy.conflist b/cni/azure-windows-multitenancy.conflist index d80ae3181b..38833b8320 100644 --- a/cni/azure-windows-multitenancy.conflist +++ b/cni/azure-windows-multitenancy.conflist @@ -24,9 +24,6 @@ "svc.cluster.local" ] }, - windowsSettings: { - "disableLoopbackDSR": true - }, "AdditionalArgs": [ { "Name": "EndpointPolicy", diff --git a/cni/netconfig.go b/cni/netconfig.go index b95a7a6a79..371d2c1d98 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -79,7 +79,7 @@ type NetworkConfig struct { } type WindowsSettings struct { - DisableLoopbackDSR bool `json:"disableLoopbackDSR,omitempty"` + EnableLoopbackDSR bool `json:"enableLoopbackDSR,omitempty"` } type K8SPodEnvArgs struct { diff --git a/cni/network/network.go b/cni/network/network.go index 45db88c6b1..9cfced72c7 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -76,6 +76,12 @@ type NetPlugin struct { multitenancyClient MultitenancyClient } +type PolicyArgs struct { + nwInfo *network.NetworkInfo + nwCfg *cni.NetworkConfig + ipconfigs []*cniTypesCurr.IPConfig +} + // client for node network service type NnsClient interface { // Do network port programming for the pod via node network service. @@ -692,8 +698,12 @@ func (plugin *NetPlugin) createEndpointInternal(opt *createEndpointInternalOpt) err = plugin.Errorf("Failed to getEndpointDNSSettings: %v", err) return epInfo, err } - - endpointPolicies, err := getEndpointPolicies(opt.nwCfg, opt.args.Netns, opt.nwInfo, opt.result.IPs) + policyArgs := PolicyArgs{ + nwInfo: opt.nwInfo, + nwCfg: opt.nwCfg, + ipconfigs: opt.result.IPs, + } + endpointPolicies, err := getEndpointPolicies(policyArgs) if err != nil { log.Errorf("Failed to get endpoint policies:%v", err) return epInfo, err diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index e9f68b84ac..d5253b3abf 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -117,7 +117,7 @@ func getEndpointDNSSettings(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Resul return getNetworkDNSSettings(nwCfg, result) } -func getEndpointPolicies(nwCfg *cni.NetworkConfig, netns string, nwInfo *network.NetworkInfo, ipconfigs []*cniTypesCurr.IPConfig) ([]policy.Policy, error) { +func getEndpointPolicies(PolicyArgs) ([]policy.Policy, error) { return nil, nil } diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 150b134747..42d81e15f1 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -296,48 +296,54 @@ func getPoliciesFromRuntimeCfg(nwCfg *cni.NetworkConfig) []policy.Policy { return policies } -func getEndpointPolicies(nwCfg *cni.NetworkConfig, netns string, nwInfo *network.NetworkInfo, ipconfigs []*cniTypesCurr.IPConfig) ([]policy.Policy, error) { +func getEndpointPolicies(args PolicyArgs) ([]policy.Policy, error) { var policies []policy.Policy - if nwCfg.IPV6Mode == network.IPV6Nat { - ipv6Policy, err := addIPV6EndpointPolicy(nwInfo) + if args.nwCfg.IPV6Mode == network.IPV6Nat { + ipv6Policy, err := getIPV6EndpointPolicy(args.nwInfo) if err != nil { - return nil, errors.Wrap(err, "failed to set ipv6 endpoint policy") + return nil, errors.Wrap(err, "failed to get ipv6 endpoint policy") } policies = append(policies, ipv6Policy) } - if !nwCfg.WindowsSettings.DisableLoopbackDSR { - if isv2, _ := network.UseHnsV2(netns); !isv2 { - log.Printf("hnsv2 not supported. so skip adding dsr policy") - return nil, nil + if args.nwCfg.WindowsSettings.EnableLoopbackDSR { + dsrPolicies, err := getLoopbackDSRPolicy(args) + if err != nil { + return nil, errors.Wrap(err, "failed to get loopback dsr policy") } - for _, config := range ipconfigs { - // consider DSR policy only for ipv4 address. Add for ipv6 when required - if config.Address.IP.To4() != nil { - dsrData := policy.LoopbackDSR{ - Type: policy.LoopbackDSRPolicy, - IPAddress: config.Address.IP, - } - - dsrDataBytes, err := json.Marshal(dsrData) - if err != nil { - return nil, errors.Wrap(err, "failed to marshal dsr data") - } - dsrPolicy := policy.Policy{ - Type: policy.EndpointPolicy, - Data: dsrDataBytes, - } - policies = append(policies, dsrPolicy) + policies = append(policies, dsrPolicies...) + } + + return policies, nil +} + +func getLoopbackDSRPolicy(args PolicyArgs) ([]policy.Policy, error) { + var policies []policy.Policy + for _, config := range args.ipconfigs { + // consider DSR policy only for ipv4 address. Add for ipv6 when required + if config.Address.IP.To4() != nil { + dsrData := policy.LoopbackDSR{ + Type: policy.LoopbackDSRPolicy, + IPAddress: config.Address.IP, } - } + dsrDataBytes, err := json.Marshal(dsrData) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal dsr data") + } + dsrPolicy := policy.Policy{ + Type: policy.EndpointPolicy, + Data: dsrDataBytes, + } + policies = append(policies, dsrPolicy) + } } return policies, nil } -func addIPV6EndpointPolicy(nwInfo *network.NetworkInfo) (policy.Policy, error) { +func getIPV6EndpointPolicy(nwInfo *network.NetworkInfo) (policy.Policy, error) { var eppolicy policy.Policy if len(nwInfo.Subnets) < 2 { diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go index 6cace557e9..0782637dd8 100644 --- a/network/policy/policy_windows.go +++ b/network/policy/policy_windows.go @@ -66,7 +66,9 @@ func SerializePolicies(policyType CNIPolicyType, policies []Policy, epInfoData m for _, policy := range policies { if policy.Type == policyType { - if isPolicyTypeOutBoundNAT := IsPolicyTypeOutBoundNAT(policy); isPolicyTypeOutBoundNAT { + endpointPolicyType := GetPolicyType(policy) + switch endpointPolicyType { + case OutBoundNatPolicy: if snatAndSerialize || !enableMultiTenancy { if serializedOutboundNatPolicy, err := SerializeOutBoundNATPolicy(policy, epInfoData); err != nil { log.Printf("Failed to serialize OutBoundNAT policy") @@ -74,14 +76,20 @@ func SerializePolicies(policyType CNIPolicyType, policies []Policy, epInfoData m jsonPolicies = append(jsonPolicies, serializedOutboundNatPolicy) } } - } else if isPolicyTypeNAT := IsPolicyTypeNAT(policy); isPolicyTypeNAT { + case PortMappingPolicy: // NATPolicy comes as a HNSv2 type, it needs to be converted to HNSv1 if serializedNatPolicy, err := SerializeNATPolicy(policy); err != nil { log.Printf("Failed to serialize NatPolicy") } else { jsonPolicies = append(jsonPolicies, serializedNatPolicy) } - } else { + case LoopbackDSRPolicy: + if dsrPolicy, err := SerializeLoopbackDSRPolicy(policy); err != nil { + log.Printf("Failed to serialize DSR policy") + } else { + jsonPolicies = append(jsonPolicies, dsrPolicy) + } + default: jsonPolicies = append(jsonPolicies, policy.Data) } } @@ -118,36 +126,6 @@ func GetOutBoundNatExceptionList(policy Policy) ([]string, error) { return nil, nil } -// IsPolicyTypeOutBoundNAT return true if the policy type is OutBoundNAT -func IsPolicyTypeOutBoundNAT(policy Policy) bool { - if policy.Type == EndpointPolicy { - var data KVPairOutBoundNAT - if err := json.Unmarshal(policy.Data, &data); err != nil { - return false - } - - if data.Type == OutBoundNatPolicy { - return true - } - } - - return false -} - -// IsPolicyTypeNAT returns true if the policy type is NAT -func IsPolicyTypeNAT(policy Policy) bool { - if policy.Type == EndpointPolicy { - var endpointPolicy hcn.EndpointPolicy - if err := json.Unmarshal(policy.Data, &endpointPolicy); err != nil { - return false - } - if endpointPolicy.Type == hcn.PortMapping { - return true - } - } - return false -} - func SerializeNATPolicy(policy Policy) (json.RawMessage, error) { var ( endpointPolicy hcn.EndpointPolicy @@ -206,6 +184,32 @@ func SerializeOutBoundNATPolicy(policy Policy, epInfoData map[string]interface{} return nil, fmt.Errorf("OutBoundNAT policy not set") } +func SerializeLoopbackDSRPolicy(policy Policy) (json.RawMessage, error) { + var dsrData LoopbackDSR + + if err := hcn.DSRSupported(); err != nil { + return []byte{}, errors.Wrap(err, "hcn doesn't support DSR") + } + + if err := json.Unmarshal(policy.Data, &dsrData); err != nil { + return []byte{}, errors.Wrap(err, "unmarshal dsr data failed") + } + + log.Printf("DSR policy for ip:%s", dsrData.IPAddress.String()) + + hnsLoopbackRoute := hcsshim.OutboundNatPolicy{ + Policy: hcsshim.Policy{Type: hcsshim.OutboundNat}, + Destinations: []string{dsrData.IPAddress.String()}, + } + + rawPolicy, err := json.Marshal(hnsLoopbackRoute) + if err != nil { + return []byte{}, errors.Wrap(err, "marshal error for loopbackdsr policy") + } + + return rawPolicy, nil +} + // GetPolicyType parses the policy and returns the policy type func GetPolicyType(policy Policy) CNIPolicyType { // Check if the type is OutBoundNAT @@ -215,7 +219,6 @@ func GetPolicyType(policy Policy) CNIPolicyType { return OutBoundNatPolicy } } - // Check if the type is Route var dataRoute KVPairRoute if err := json.Unmarshal(policy.Data, &dataRoute); err == nil { @@ -449,8 +452,7 @@ func GetHcnLoopbackDSRPolicy(policy Policy) (hcn.EndpointPolicy, error) { return hcn.EndpointPolicy{}, errors.Wrap(err, "hcn doesn't support DSR") } - err := json.Unmarshal(policy.Data, &dsrData) - if err != nil { + if err := json.Unmarshal(policy.Data, &dsrData); err != nil { return hcn.EndpointPolicy{}, errors.Wrap(err, "unmarshal dsr data failed") } From 9f5c4f45dfdaba5b611abe0d208231f063c6c601 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 24 Nov 2021 16:09:33 -0800 Subject: [PATCH 5/5] fixed windows uts --- cni/network/network_windows_test.go | 54 +++++++++++++++-------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/cni/network/network_windows_test.go b/cni/network/network_windows_test.go index 7f5a506426..32529ba76a 100644 --- a/cni/network/network_windows_test.go +++ b/cni/network/network_windows_test.go @@ -51,7 +51,7 @@ func TestAddWithRunTimeNetPolicies(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - p, err := addIPV6EndpointPolicy(&tt.nwInfo) + p, err := getIPV6EndpointPolicy(&tt.nwInfo) if tt.wantErr { require.Error(t, err) } else { @@ -247,39 +247,41 @@ func TestSetPoliciesFromNwCfg(t *testing.T) { func TestDSRPolciy(t *testing.T) { tests := []struct { name string - nwCfg cni.NetworkConfig - nwInfo network.NetworkInfo - ipconfig []*cniTypesCurr.IPConfig + args PolicyArgs wantCount int }{ { - name: "test enable dsr policy", - nwCfg: cni.NetworkConfig{}, - nwInfo: network.NetworkInfo{}, - ipconfig: []*cniTypesCurr.IPConfig{ - { - Address: func() net.IPNet { - _, ipnet, _ := net.ParseCIDR("10.0.0.5/24") - return *ipnet - }(), + name: "test enable dsr policy", + args: PolicyArgs{ + nwCfg: &cni.NetworkConfig{ + WindowsSettings: cni.WindowsSettings{ + EnableLoopbackDSR: true, + }, + }, + nwInfo: &network.NetworkInfo{}, + ipconfigs: []*cniTypesCurr.IPConfig{ + { + Address: func() net.IPNet { + _, ipnet, _ := net.ParseCIDR("10.0.0.5/24") + return *ipnet + }(), + }, }, }, wantCount: 1, }, { name: "test disable dsr policy", - nwCfg: cni.NetworkConfig{ - WindowsSettings: cni.WindowsSettings{ - DisableLoopbackDSR: true, - }, - }, - nwInfo: network.NetworkInfo{}, - ipconfig: []*cniTypesCurr.IPConfig{ - { - Address: func() net.IPNet { - _, ipnet, _ := net.ParseCIDR("10.0.0.5/24") - return *ipnet - }(), + args: PolicyArgs{ + nwCfg: &cni.NetworkConfig{}, + nwInfo: &network.NetworkInfo{}, + ipconfigs: []*cniTypesCurr.IPConfig{ + { + Address: func() net.IPNet { + _, ipnet, _ := net.ParseCIDR("10.0.0.5/24") + return *ipnet + }(), + }, }, }, wantCount: 0, @@ -289,7 +291,7 @@ func TestDSRPolciy(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - policies, err := getEndpointPolicies(&tt.nwCfg, &tt.nwInfo, tt.ipconfig) + policies, err := getEndpointPolicies(tt.args) require.NoError(t, err) require.Equal(t, tt.wantCount, len(policies)) })