From 79852d0b0f21b7899b14a64cff5890abbe437ef3 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 1 Nov 2023 13:05:48 -0400 Subject: [PATCH 1/4] chores: chage nodeCIDRs to infraVNETCIDRs --- cns/configuration/env.go | 22 +++++++++--------- cns/configuration/env_test.go | 10 ++++++++ cns/middlewares/mock/mockSWIFTv2.go | 36 ++++++++++++++--------------- cns/middlewares/swiftV2.go | 30 ++++++++++++------------ cns/middlewares/swiftV2_test.go | 4 ++-- 5 files changed, 56 insertions(+), 46 deletions(-) diff --git a/cns/configuration/env.go b/cns/configuration/env.go index 6b16668cb2..cf718a442b 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -14,10 +14,10 @@ const ( // LabelNodeSwiftV2 is the Node label for Swift V2 LabelNodeSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled" // LabelPodSwiftV2 is the Pod label for Swift V2 - LabelPodSwiftV2 = "kubernetes.azure.com/pod-network" - EnvPodCIDRs = "POD_CIDRs" - EnvServiceCIDRs = "SERVICE_CIDRs" - EnvNodeCIDRs = "NODE_CIDRs" + LabelPodSwiftV2 = "kubernetes.azure.com/pod-network" + EnvPodCIDRs = "POD_CIDRs" + EnvServiceCIDRs = "SERVICE_CIDRs" + EnvInfraVNETCIDRs = "INFRA_VNET_CIDRs" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. @@ -29,8 +29,8 @@ var ErrPodCIDRsUnset = errors.Errorf("must declare %s environment variable", Env // ErrServiceCIDRsUnset indicates the the $EnvServiceCIDRs variable is unset in the environment. var ErrServiceCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvServiceCIDRs) -// ErrNodeCIDRsUnset indicates the the $EnvNodeCIDRs variable is unset in the environment. -var ErrNodeCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvNodeCIDRs) +// ErrInfraVNETCIDRsUnset indicates the the $EnvInfraVNETCIDRs variable is unset in the environment. +var ErrInfraVNETCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvInfraVNETCIDRs) // NodeName checks the environment variables for the NODENAME and returns it or an error if unset. func NodeName() (string, error) { @@ -62,10 +62,10 @@ func ServiceCIDRs() (string, error) { return serviceCIDRs, nil } -func NodeCIDRs() (string, error) { - nodeCIDRs := os.Getenv(EnvNodeCIDRs) - if nodeCIDRs == "" { - return "", ErrNodeCIDRsUnset +func InfraVNETCIDRs() (string, error) { + infraVNETCIDRs := os.Getenv(EnvInfraVNETCIDRs) + if infraVNETCIDRs == "" { + return "", ErrInfraVNETCIDRsUnset } - return nodeCIDRs, nil + return infraVNETCIDRs, nil } diff --git a/cns/configuration/env_test.go b/cns/configuration/env_test.go index cc411ec583..2bca096cde 100644 --- a/cns/configuration/env_test.go +++ b/cns/configuration/env_test.go @@ -37,3 +37,13 @@ func TestServiceCIDRs(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "test", cidr) } + +func TestInfraVNETCIDRs(t *testing.T) { + _, err := InfraVNETCIDRs() + require.Error(t, err) + require.ErrorIs(t, err, ErrInfraVNETCIDRsUnset) + os.Setenv(EnvInfraVNETCIDRs, "test") + cidr, err := InfraVNETCIDRs() + assert.NoError(t, err) + assert.Equal(t, "test", cidr) +} diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 11d1266f76..78ffd8ea2a 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -56,7 +56,7 @@ func (m *SWIFTv2Middleware) SetMTPNCReady() { func (m *SWIFTv2Middleware) SetEnvVar() { os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24") os.Setenv(configuration.EnvServiceCIDRs, "10.0.2.10/24") - os.Setenv(configuration.EnvNodeCIDRs, "10.0.3.10/24") + os.Setenv(configuration.EnvInfraVNETCIDRs, "10.0.3.10/24") } func (m *SWIFTv2Middleware) UnsetEnvVar() error { @@ -66,8 +66,8 @@ func (m *SWIFTv2Middleware) UnsetEnvVar() error { if err := os.Unsetenv(configuration.EnvServiceCIDRs); err != nil { return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvServiceCIDRs, err) } - if err := os.Unsetenv(configuration.EnvNodeCIDRs); err != nil { - return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvNodeCIDRs, err) + if err := os.Unsetenv(configuration.EnvInfraVNETCIDRs); err != nil { + return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvInfraVNETCIDRs, err) } return nil } @@ -136,14 +136,14 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = []cns.Route{route} case cns.InfraNIC: - // Get and parse nodeCIDRs from env - nodeCIDRs, err := configuration.NodeCIDRs() + // Get and parse infraVNETCIDRs from env + infraVNETCIDRs, err := configuration.InfraVNETCIDRs() if err != nil { - return errors.Wrapf(err, "failed to get nodeCIDR from env") + return errors.Wrapf(err, "failed to get infraVNETCIDRs from env") } - nodeCIDRsv4, nodeCIDRsv6, err := utils.ParseCIDRs(nodeCIDRs) + infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs) if err != nil { - return errors.Wrapf(err, "failed to parse nodeCIDRs") + return errors.Wrapf(err, "failed to parse infraVNETCIDRs") } // Get and parse podCIDRs from env @@ -187,13 +187,13 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route) } - // route for IPv4 nodeCIDR traffic - for _, nodeCIDRv4 := range nodeCIDRsv4 { - nodeCIDRv4Route := cns.Route{ - IPAddress: nodeCIDRv4, + // route for IPv4 infraVNETCIDR traffic + for _, infraVNETCIDRsv4 := range infraVNETCIDRsv4 { + infraVNETCIDRsv4Route := cns.Route{ + IPAddress: infraVNETCIDRsv4, GatewayIPAddress: overlayGatewayv4, } - podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv4Route) + podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRsv4Route) } } else { // routes for IPv6 podCIDR traffic @@ -212,13 +212,13 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route) } - // route for IPv6 nodeCIDR traffic - for _, nodeCIDRv6 := range nodeCIDRsv6 { - nodeCIDRv6Route := cns.Route{ - IPAddress: nodeCIDRv6, + // route for IPv6 infraVNETCIDR traffic + for _, infraVNETCIDRv6 := range infraVNETCIDRsv6 { + infraVNETCIDRv6Route := cns.Route{ + IPAddress: infraVNETCIDRv6, GatewayIPAddress: overlayGatewayV6, } - podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv6Route) + podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv6Route) } } podIPInfo.SkipDefaultRoutes = true diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 7326ccc6bd..cbe3534c20 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -120,14 +120,14 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = []cns.Route{route} case cns.InfraNIC: - // Get and parse nodeCIDRs from env - nodeCIDRs, err := configuration.NodeCIDRs() + // Get and parse infraVNETCIDRs from env + infraVNETCIDRs, err := configuration.InfraVNETCIDRs() if err != nil { - return errors.Wrapf(err, "failed to get nodeCIDR from env") + return errors.Wrapf(err, "failed to get infraVNETCIDRs from env") } - nodeCIDRsv4, nodeCIDRsv6, err := utils.ParseCIDRs(nodeCIDRs) + infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs) if err != nil { - return errors.Wrapf(err, "failed to parse nodeCIDRs") + return errors.Wrapf(err, "failed to parse infraVNETCIDRs") } // Get and parse podCIDRs from env @@ -171,13 +171,13 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route) } - // route for IPv4 nodeCIDR traffic - for _, nodeCIDRv4 := range nodeCIDRsv4 { - nodeCIDRv4Route := cns.Route{ - IPAddress: nodeCIDRv4, + // route for IPv4 infraVNETCIDR traffic + for _, infraVNETCIDRv4 := range infraVNETCIDRsv4 { + infraVNETCIDRv4Route := cns.Route{ + IPAddress: infraVNETCIDRv4, GatewayIPAddress: overlayGatewayv4, } - podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv4Route) + podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv4Route) } } else { // routes for IPv6 podCIDR traffic @@ -196,13 +196,13 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route) } - // route for IPv6 nodeCIDR traffic - for _, nodeCIDRv6 := range nodeCIDRsv6 { - nodeCIDRv6Route := cns.Route{ - IPAddress: nodeCIDRv6, + // route for IPv6 infraVNETCIDR traffic + for _, infraVNETCIDRv6 := range infraVNETCIDRsv6 { + infraVNETCIDRv6Route := cns.Route{ + IPAddress: infraVNETCIDRv6, GatewayIPAddress: overlayGatewayV6, } - podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv6Route) + podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv6Route) } } podIPInfo.SkipDefaultRoutes = true diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 74a2daf246..fda084300d 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -30,13 +30,13 @@ var ( func setEnvVar() { os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24,16A0:0010:AB00:001E::2/32") os.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16,16A0:0010:AB00:0000::/32") - os.Setenv(configuration.EnvNodeCIDRs, "10.240.0.1/16,16A0:0020:AB00:0000::/32") + os.Setenv(configuration.EnvInfraVNETCIDRs, "10.240.0.1/16,16A0:0020:AB00:0000::/32") } func unsetEnvVar() { os.Unsetenv(configuration.EnvPodCIDRs) os.Unsetenv(configuration.EnvServiceCIDRs) - os.Unsetenv(configuration.EnvNodeCIDRs) + os.Unsetenv(configuration.EnvInfraVNETCIDRs) } func TestMain(m *testing.M) { From 49db65be3aeee26d83539b7860d27ea15614f747 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 1 Nov 2023 13:23:17 -0400 Subject: [PATCH 2/4] chores: add routes for virtualGW --- cns/middlewares/mock/mockSWIFTv2.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 78ffd8ea2a..7db687cd7d 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -25,6 +25,7 @@ var ( const ( prefixLength = 32 overlayGatewayv4 = "169.254.1.1" + virtualGW = "169.254.2.1" overlayGatewayV6 = "fe80::1234:5678:9abc" ) @@ -130,11 +131,15 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { podIPInfo.Routes = []cns.Route{} switch podIPInfo.NICType { case cns.DelegatedVMNIC: + virtualGWRoute := cns.Route{ + IPAddress: virtualGW, + } // default route via SWIFT v2 interface route := cns.Route{ - IPAddress: "0.0.0.0/0", + IPAddress: "0.0.0.0/0", + GatewayIPAddress: virtualGW, } - podIPInfo.Routes = []cns.Route{route} + podIPInfo.Routes = []cns.Route{virtualGWRoute, route} case cns.InfraNIC: // Get and parse infraVNETCIDRs from env infraVNETCIDRs, err := configuration.InfraVNETCIDRs() From db7fb9a998b3635e8407ab9effbd404539aa2323 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 1 Nov 2023 13:56:37 -0400 Subject: [PATCH 3/4] fix: add /32 prefix --- cns/middlewares/mock/mockSWIFTv2.go | 2 +- cns/middlewares/swiftV2.go | 9 +++++++-- cns/middlewares/swiftV2_test.go | 6 +++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 7db687cd7d..b1d30b2fcf 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -25,7 +25,7 @@ var ( const ( prefixLength = 32 overlayGatewayv4 = "169.254.1.1" - virtualGW = "169.254.2.1" + virtualGW = "169.254.2.1/32" overlayGatewayV6 = "fe80::1234:5678:9abc" ) diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index cbe3534c20..03cecf8b7d 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -26,6 +26,7 @@ var ( const ( prefixLength = 32 overlayGatewayv4 = "169.254.1.1" + virtualGW = "169.254.2.1/32" overlayGatewayV6 = "fe80::1234:5678:9abc" ) @@ -114,11 +115,15 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { podIPInfo.Routes = []cns.Route{} switch podIPInfo.NICType { case cns.DelegatedVMNIC: + virtualGWRoute := cns.Route{ + IPAddress: virtualGW, + } // default route via SWIFT v2 interface route := cns.Route{ - IPAddress: "0.0.0.0/0", + IPAddress: "0.0.0.0/0", + GatewayIPAddress: virtualGW, } - podIPInfo.Routes = []cns.Route{route} + podIPInfo.Routes = []cns.Route{virtualGWRoute, route} case cns.InfraNIC: // Get and parse infraVNETCIDRs from env infraVNETCIDRs, err := configuration.InfraVNETCIDRs() diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index fda084300d..cf513334f1 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -200,7 +200,11 @@ func TestSetRoutesSuccess(t *testing.T) { MacAddress: "12:34:56:78:9a:bc", Routes: []cns.Route{ { - IPAddress: "0.0.0.0/0", + IPAddress: virtualGW, + }, + { + IPAddress: "0.0.0.0/0", + GatewayIPAddress: virtualGW, }, }, }, From 2853207cb1c35f50eb936eecd858fe06f7ef9586 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 1 Nov 2023 14:07:49 -0400 Subject: [PATCH 4/4] fix: manually fmt.Sprintf /32 prefix with virtual gw --- cns/middlewares/mock/mockSWIFTv2.go | 4 ++-- cns/middlewares/swiftV2.go | 4 ++-- cns/middlewares/swiftV2_test.go | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index b1d30b2fcf..6b5bbeb22c 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -25,7 +25,7 @@ var ( const ( prefixLength = 32 overlayGatewayv4 = "169.254.1.1" - virtualGW = "169.254.2.1/32" + virtualGW = "169.254.2.1" overlayGatewayV6 = "fe80::1234:5678:9abc" ) @@ -132,7 +132,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { switch podIPInfo.NICType { case cns.DelegatedVMNIC: virtualGWRoute := cns.Route{ - IPAddress: virtualGW, + IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength), } // default route via SWIFT v2 interface route := cns.Route{ diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 03cecf8b7d..5c3a48dc70 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -26,7 +26,7 @@ var ( const ( prefixLength = 32 overlayGatewayv4 = "169.254.1.1" - virtualGW = "169.254.2.1/32" + virtualGW = "169.254.2.1" overlayGatewayV6 = "fe80::1234:5678:9abc" ) @@ -116,7 +116,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { switch podIPInfo.NICType { case cns.DelegatedVMNIC: virtualGWRoute := cns.Route{ - IPAddress: virtualGW, + IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength), } // default route via SWIFT v2 interface route := cns.Route{ diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index cf513334f1..7dd121717b 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -2,6 +2,7 @@ package middlewares import ( "context" + "fmt" "os" "testing" @@ -200,7 +201,7 @@ func TestSetRoutesSuccess(t *testing.T) { MacAddress: "12:34:56:78:9a:bc", Routes: []cns.Route{ { - IPAddress: virtualGW, + IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength), }, { IPAddress: "0.0.0.0/0",