From efb92a690dfc2ac5ce11a535c9be5c1f9490dfa6 Mon Sep 17 00:00:00 2001 From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Date: Wed, 17 Mar 2021 15:46:24 -0700 Subject: [PATCH] Multi card support - Prevent route override for primary ENI across multi-cards ENAs (#1396) * Multi card support * Updated filterunmanaged ENIs to account for both unmanaged and CNI unmanaged * Missed commiting this * Fixed infinite retries * Debug logs for CIDR not null scenarios * pr review --- go.mod | 4 +- go.sum | 6 +++ pkg/awsutils/awsutils.go | 54 ++++++++++++++++++------ pkg/awsutils/awsutils_test.go | 10 ++--- pkg/awsutils/mocks/awsutils_mocks.go | 47 +++++++++++++++++++-- pkg/ec2wrapper/mocks/ec2wrapper_mocks.go | 3 +- pkg/ipamd/ipamd.go | 25 ++++++++++- pkg/ipamd/ipamd_test.go | 17 ++++++++ pkg/networkutils/network.go | 3 ++ 9 files changed, 140 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index b87a06c2d9..451073a846 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/aws/amazon-vpc-cni-k8s go 1.14 require ( - github.com/aws/aws-sdk-go v1.35.27 + github.com/aws/aws-sdk-go v1.37.23 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect github.com/containernetworking/cni v0.8.0 github.com/containernetworking/plugins v0.9.0 @@ -37,7 +37,7 @@ require ( go.uber.org/zap v1.15.0 golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect golang.org/x/mod v0.4.0 // indirect - golang.org/x/net v0.0.0-20201021035429-f5854403a974 + golang.org/x/net v0.0.0-20201110031124-69a78807bb2b golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect golang.org/x/tools v0.0.0-20210113180300-f96436850f18 // indirect diff --git a/go.sum b/go.sum index 33c9647c15..afe8cd642c 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/aws/aws-sdk-go v1.35.27 h1:F0dUW+kouzchjt4X6kYfYMw1YtQPkA4pihpCDqQMrq8= github.com/aws/aws-sdk-go v1.35.27/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= +github.com/aws/aws-sdk-go v1.37.23 h1:bO80NcSmRv52w+GFpBegoLdlP/Z0OwUqQ9bbeCLCy/0= +github.com/aws/aws-sdk-go v1.37.23/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= @@ -69,6 +71,7 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -231,6 +234,7 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -307,6 +311,7 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -330,6 +335,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index 5e8942cad4..48d121f49e 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -155,6 +155,16 @@ type APIs interface { // WaitForENIAndIPsAttached waits until the ENI has been attached and the secondary IPs have been added WaitForENIAndIPsAttached(eni string, wantedSecondaryIPs int) (ENIMetadata, error) + + //SetCNIunmanaged ENI + SetCNIUnmanagedENIs(eniID []string) error + + //isCNIUnmanagedENI + IsCNIUnmanagedENI(eniID string) bool + + //RefreshSGIDs + RefreshSGIDs(mac string) error + } // EC2InstanceMetadataCache caches instance metadata @@ -171,6 +181,7 @@ type EC2InstanceMetadataCache struct { region string unmanagedENIs StringSet useCustomNetworking bool + cniunmanagedENIs StringSet imds TypedIMDS ec2SVC ec2wrapper.EC2 @@ -219,6 +230,7 @@ type DescribeAllENIsResult struct { TagMap map[string]TagMap TrunkENI string EFAENIs map[string]bool + MultiCardENIIDs []string } // msSince returns milliseconds since start. @@ -397,17 +409,7 @@ func (cache *EC2InstanceMetadataCache) initWithEC2Metadata(ctx context.Context) return err } log.Debugf("Found subnet-id: %s ", cache.subnetID) - - // retrieve security groups - err = cache.refreshSGIDs(mac) - if err != nil { - return err - } - - // Refresh security groups and VPC CIDR blocks in the background - // Ignoring errors since we will retry in 30s - go wait.Forever(func() { _ = cache.refreshSGIDs(mac) }, 30*time.Second) - + // We use the ctx here for testing, since we spawn go-routines above which will run forever. select { case <-ctx.Done(): @@ -417,8 +419,8 @@ func (cache *EC2InstanceMetadataCache) initWithEC2Metadata(ctx context.Context) return nil } -// refreshSGIDs retrieves security groups -func (cache *EC2InstanceMetadataCache) refreshSGIDs(mac string) error { +// RefreshSGIDs retrieves security groups +func (cache *EC2InstanceMetadataCache) RefreshSGIDs(mac string) error { ctx := context.TODO() sgIDs, err := cache.imds.GetSecurityGroupIDs(ctx, mac) @@ -457,7 +459,8 @@ func (cache *EC2InstanceMetadataCache) refreshSGIDs(mac string) error { newENIs := StringSet{} newENIs.Set(eniIDs) - filteredENIs := newENIs.Difference(&cache.unmanagedENIs) + tempfilteredENIs := newENIs.Difference(&cache.cniunmanagedENIs) + filteredENIs := tempfilteredENIs.Difference(&cache.unmanagedENIs) sgIDsPtrs := aws.StringSlice(sgIDs) // This will update SG for managed ENIs created by EKS. @@ -1034,13 +1037,19 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult, // Collect ENI response into ENI metadata and tags. var trunkENI string + var multiCardENIIDs []string efaENIs := make(map[string]bool, 0) tagMap := make(map[string]TagMap, len(ec2Response.NetworkInterfaces)) for _, ec2res := range ec2Response.NetworkInterfaces { + log.Infof("Got network cardindex %v for ENI %v", aws.Int64Value(ec2res.Attachment.NetworkCardIndex), aws.StringValue(ec2res.NetworkInterfaceId)) if ec2res.Attachment != nil && aws.Int64Value(ec2res.Attachment.DeviceIndex) == 0 && !aws.BoolValue(ec2res.Attachment.DeleteOnTermination) { log.Warn("Primary ENI will not get deleted when node terminates because 'delete_on_termination' is set to false") } eniID := aws.StringValue(ec2res.NetworkInterfaceId) + if aws.Int64Value(ec2res.Attachment.NetworkCardIndex) > 0 { + multiCardENIIDs = append(multiCardENIIDs, eniID) + } + eniMetadata := eniMap[eniID] interfaceType := aws.StringValue(ec2res.InterfaceType) @@ -1065,6 +1074,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult, TagMap: tagMap, TrunkENI: trunkENI, EFAENIs: efaENIs, + MultiCardENIIDs: multiCardENIIDs, }, nil } @@ -1507,3 +1517,19 @@ func (cache *EC2InstanceMetadataCache) getENIsFromPaginatedDescribeNetworkInterf } return innerErr } + +//SetCNIUnmanagedENIs Set unmanaged ENI set +func (cache *EC2InstanceMetadataCache) SetCNIUnmanagedENIs(eniID []string) error { + if len(eniID) != 0 { + cache.cniunmanagedENIs.Set(eniID) + } + return nil +} + +//IsCNIUnmanagedENI returns if the eni is unmanaged +func (cache *EC2InstanceMetadataCache) IsCNIUnmanagedENI(eniID string) bool { + if len(eniID) != 0 { + return cache.cniunmanagedENIs.Has(eniID) + } + return false +} \ No newline at end of file diff --git a/pkg/awsutils/awsutils_test.go b/pkg/awsutils/awsutils_test.go index be6e594407..69b253d731 100644 --- a/pkg/awsutils/awsutils_test.go +++ b/pkg/awsutils/awsutils_test.go @@ -110,8 +110,6 @@ func TestInitWithEC2metadata(t *testing.T) { defer ctrl.Finish() mockMetadata := testMetadata(nil) - mockEC2.EXPECT().ModifyNetworkInterfaceAttributeWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) - ins := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2} err := ins.initWithEC2Metadata(ctx) if assert.NoError(t, err) { @@ -120,7 +118,6 @@ func TestInitWithEC2metadata(t *testing.T) { assert.Equal(t, ins.instanceID, instanceID) assert.Equal(t, ins.primaryENImac, primaryMAC) assert.Equal(t, ins.primaryENI, primaryeniID) - assert.Equal(t, len(ins.securityGroups.SortedList()), 2) assert.Equal(t, subnetID, ins.subnetID) } } @@ -132,8 +129,6 @@ func TestInitWithEC2metadataErr(t *testing.T) { ctrl, mockEC2 := setup(t) defer ctrl.Finish() - mockEC2.EXPECT().ModifyNetworkInterfaceAttributeWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) - var keys []string for k := range testMetadata(nil) { keys = append(keys, k) @@ -275,6 +270,9 @@ func TestDescribeAllENIs(t *testing.T) { TagSet: []*ec2.Tag{ {Key: aws.String("foo"), Value: aws.String("foo-value")}, }, + Attachment: &ec2.NetworkInterfaceAttachment{ + NetworkCardIndex: aws.Int64(0), + }, }}, } @@ -313,8 +311,6 @@ func TestTagEni(t *testing.T) { defer ctrl.Finish() mockMetadata := testMetadata(nil) - mockEC2.EXPECT().ModifyNetworkInterfaceAttributeWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) - ins := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2} err := ins.initWithEC2Metadata(ctx) diff --git a/pkg/awsutils/mocks/awsutils_mocks.go b/pkg/awsutils/mocks/awsutils_mocks.go index 79479e5528..3d320222dd 100644 --- a/pkg/awsutils/mocks/awsutils_mocks.go +++ b/pkg/awsutils/mocks/awsutils_mocks.go @@ -19,12 +19,11 @@ package mock_awsutils import ( - net "net" - reflect "reflect" - awsutils "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" ec2 "github.com/aws/aws-sdk-go/service/ec2" gomock "github.com/golang/mock/gomock" + net "net" + reflect "reflect" ) // MockAPIs is a mock of APIs interface @@ -253,6 +252,20 @@ func (mr *MockAPIsMockRecorder) GetVPCIPv4CIDRs() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVPCIPv4CIDRs", reflect.TypeOf((*MockAPIs)(nil).GetVPCIPv4CIDRs)) } +// IsCNIUnmanagedENI mocks base method +func (m *MockAPIs) IsCNIUnmanagedENI(arg0 string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsCNIUnmanagedENI", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsCNIUnmanagedENI indicates an expected call of IsCNIUnmanagedENI +func (mr *MockAPIsMockRecorder) IsCNIUnmanagedENI(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsCNIUnmanagedENI", reflect.TypeOf((*MockAPIs)(nil).IsCNIUnmanagedENI), arg0) +} + // IsUnmanagedENI mocks base method func (m *MockAPIs) IsUnmanagedENI(arg0 string) bool { m.ctrl.T.Helper() @@ -267,6 +280,34 @@ func (mr *MockAPIsMockRecorder) IsUnmanagedENI(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUnmanagedENI", reflect.TypeOf((*MockAPIs)(nil).IsUnmanagedENI), arg0) } +// RefreshSGIDs mocks base method +func (m *MockAPIs) RefreshSGIDs(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RefreshSGIDs", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// RefreshSGIDs indicates an expected call of RefreshSGIDs +func (mr *MockAPIsMockRecorder) RefreshSGIDs(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshSGIDs", reflect.TypeOf((*MockAPIs)(nil).RefreshSGIDs), arg0) +} + +// SetCNIUnmanagedENIs mocks base method +func (m *MockAPIs) SetCNIUnmanagedENIs(arg0 []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetCNIUnmanagedENIs", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetCNIUnmanagedENIs indicates an expected call of SetCNIUnmanagedENIs +func (mr *MockAPIsMockRecorder) SetCNIUnmanagedENIs(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetCNIUnmanagedENIs", reflect.TypeOf((*MockAPIs)(nil).SetCNIUnmanagedENIs), arg0) +} + // SetUnmanagedENIs mocks base method func (m *MockAPIs) SetUnmanagedENIs(arg0 []string) { m.ctrl.T.Helper() diff --git a/pkg/ec2wrapper/mocks/ec2wrapper_mocks.go b/pkg/ec2wrapper/mocks/ec2wrapper_mocks.go index f4791e0abd..5ab360e1fd 100644 --- a/pkg/ec2wrapper/mocks/ec2wrapper_mocks.go +++ b/pkg/ec2wrapper/mocks/ec2wrapper_mocks.go @@ -20,11 +20,10 @@ package mock_ec2wrapper import ( context "context" - reflect "reflect" - request "github.com/aws/aws-sdk-go/aws/request" ec2 "github.com/aws/aws-sdk-go/service/ec2" gomock "github.com/golang/mock/gomock" + reflect "reflect" ) // MockEC2 is a mock of EC2 interface diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 5acbae425b..9237ceee15 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -311,6 +311,17 @@ func New(k8sapiClient kubernetes.Interface, eniConfig *eniconfig.ENIConfigContro if err != nil { return nil, err } + + mac := c.awsClient.GetPrimaryENImac() + // retrieve security groups + err = c.awsClient.RefreshSGIDs(mac) + if err != nil { + return nil, err + } + + // Refresh security groups and VPC CIDR blocks in the background + // Ignoring errors since we will retry in 30s + go wait.Forever(func() { _ = c.awsClient.RefreshSGIDs(mac) }, 30*time.Second) return c, nil } @@ -347,15 +358,21 @@ func (c *IPAMContext) nodeInit() error { return errors.New("ipamd init: failed to retrieve attached ENIs info") } log.Debugf("DescribeAllENIs success: ENIs: %d, tagged: %d", len(metadataResult.ENIMetadata), len(metadataResult.TagMap)) + c.awsClient.SetCNIUnmanagedENIs(metadataResult.MultiCardENIIDs) c.setUnmanagedENIs(metadataResult.TagMap) enis := c.filterUnmanagedENIs(metadataResult.ENIMetadata) for _, eni := range enis { log.Debugf("Discovered ENI %s, trying to set it up", eni.ENIID) // Retry ENI sync + if c.awsClient.IsCNIUnmanagedENI(eni.ENIID) { + log.Infof("Skipping ENI %s since it is not on network card 0", eni.ENIID) + continue + } retry := 0 for { retry++ + if err = c.setupENI(eni.ENIID, eni, eni.ENIID == metadataResult.TrunkENI, metadataResult.EFAENIs[eni.ENIID]); err == nil { log.Infof("ENI %s set up.", eni.ENIID) break @@ -969,6 +986,7 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { // Just copy values of the EFA set efaENIs = metadataResult.EFAENIs c.setUnmanagedENIs(metadataResult.TagMap) + c.awsClient.SetCNIUnmanagedENIs(metadataResult.MultiCardENIIDs) attachedENIs = c.filterUnmanagedENIs(metadataResult.ENIMetadata) } @@ -1196,7 +1214,12 @@ func (c *IPAMContext) filterUnmanagedENIs(enis []awsutils.ENIMetadata) []awsutil log.Debugf("Skipping ENI %s: tagged with %s", eni.ENIID, eniNoManageTagKey) numFiltered++ continue + } else if c.awsClient.IsCNIUnmanagedENI(eni.ENIID) { + log.Debugf("Skipping ENI %s: since on non-zero network card", eni.ENIID) + numFiltered++ + continue } + ret = append(ret, eni) } c.unmanagedENI = numFiltered @@ -1320,4 +1343,4 @@ func (c *IPAMContext) SetNodeLabel(key, value string) error { // GetPod returns the pod matching the name and namespace func (c *IPAMContext) GetPod(podName, namespace string) (*v1.Pod, error) { return c.k8sClient.CoreV1().Pods(namespace).Get(podName, metav1.GetOptions{}) -} +} \ No newline at end of file diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index 1c8a764367..e3cf7daf24 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -110,6 +110,8 @@ func TestNodeInit(t *testing.T) { m.awsutils.EXPECT().GetIPv4sFromEC2(eni2.ENIID).AnyTimes().Return(eni2.IPv4Addresses, nil) m.awsutils.EXPECT().IsUnmanagedENI(eni1.ENIID).Return(false).AnyTimes() m.awsutils.EXPECT().IsUnmanagedENI(eni2.ENIID).Return(false).AnyTimes() + m.awsutils.EXPECT().IsCNIUnmanagedENI(eni1.ENIID).Return(false).AnyTimes() + m.awsutils.EXPECT().IsCNIUnmanagedENI(eni2.ENIID).Return(false).AnyTimes() primaryIP := net.ParseIP(ipaddr01) m.awsutils.EXPECT().GetVPCIPv4CIDRs().AnyTimes().Return(cidrs, nil) @@ -124,10 +126,12 @@ func TestNodeInit(t *testing.T) { TagMap: map[string]awsutils.TagMap{}, TrunkENI: "", EFAENIs: make(map[string]bool), + MultiCardENIIDs: nil, } m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) + m.awsutils.EXPECT().SetCNIUnmanagedENIs(resp.MultiCardENIIDs).AnyTimes() m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP) var rules []netlink.Rule @@ -367,6 +371,7 @@ func TestNodeIPPoolReconcile(t *testing.T) { // Always the primary ENI m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) m.awsutils.EXPECT().IsUnmanagedENI(primaryENIid).AnyTimes().Return(false) + m.awsutils.EXPECT().IsCNIUnmanagedENI(primaryENIid).AnyTimes().Return(false) eniMetadataList := []awsutils.ENIMetadata{primaryENIMetadata} m.awsutils.EXPECT().GetAttachedENIs().Return(eniMetadataList, nil) resp := awsutils.DescribeAllENIsResult{ @@ -374,9 +379,11 @@ func TestNodeIPPoolReconcile(t *testing.T) { TagMap: map[string]awsutils.TagMap{}, TrunkENI: "", EFAENIs: make(map[string]bool), + MultiCardENIIDs : nil, } m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) + m.awsutils.EXPECT().SetCNIUnmanagedENIs(resp.MultiCardENIIDs).AnyTimes() mockContext.nodeIPPoolReconcile(0) curENIs := mockContext.dataStore.GetENIInfos() @@ -413,14 +420,17 @@ func TestNodeIPPoolReconcile(t *testing.T) { // Two ENIs found m.awsutils.EXPECT().GetAttachedENIs().Return(twoENIs, nil) m.awsutils.EXPECT().IsUnmanagedENI(secENIid).Times(2).Return(false) + m.awsutils.EXPECT().IsCNIUnmanagedENI(secENIid).Times(2).Return(false) resp2 := awsutils.DescribeAllENIsResult{ ENIMetadata: twoENIs, TagMap: map[string]awsutils.TagMap{}, TrunkENI: "", EFAENIs: make(map[string]bool), + MultiCardENIIDs : nil, } m.awsutils.EXPECT().DescribeAllENIs().Return(resp2, nil) m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, primarySubnet) + m.awsutils.EXPECT().SetCNIUnmanagedENIs(resp2.MultiCardENIIDs).AnyTimes() mockContext.nodeIPPoolReconcile(0) @@ -622,6 +632,12 @@ func TestIPAMContext_filterUnmanagedENIs(t *testing.T) { return false }).AnyTimes() + + mockAWSUtils.EXPECT().IsCNIUnmanagedENI(gomock.Any()).DoAndReturn( + func(eni string) (unmanaged bool) { + return false + + }).AnyTimes() if got := c.filterUnmanagedENIs(tt.enis); !reflect.DeepEqual(got, tt.want) { t.Errorf("filterUnmanagedENIs() = %v, want %v", got, tt.want) @@ -682,6 +698,7 @@ func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) { eniMetadataList := []awsutils.ENIMetadata{primaryENIMetadata} m.awsutils.EXPECT().GetAttachedENIs().Return(eniMetadataList, nil) m.awsutils.EXPECT().IsUnmanagedENI(eniID).Return(false).AnyTimes() + m.awsutils.EXPECT().IsCNIUnmanagedENI(eniID).Return(false).AnyTimes() // First reconcile, IMDS returns correct IPs so no change needed mockContext.nodeIPPoolReconcile(0) diff --git a/pkg/networkutils/network.go b/pkg/networkutils/network.go index 879423b4b7..350fff3d6b 100644 --- a/pkg/networkutils/network.go +++ b/pkg/networkutils/network.go @@ -310,9 +310,11 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr } var allCIDRs []snatCIDR for _, cidr := range vpcCIDRs { + log.Debugf("Adding %s CIDR to NAT chain", cidr) allCIDRs = append(allCIDRs, snatCIDR{cidr: cidr, isExclusion: false}) } for _, cidr := range n.excludeSNATCIDRs { + log.Debugf("Adding %s Excluded CIDR to NAT chain", cidr) allCIDRs = append(allCIDRs, snatCIDR{cidr: cidr, isExclusion: true}) } @@ -322,6 +324,7 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr return errors.Wrapf(err, "host network setup: failed to get SNAT chain rules to clear") } + log.Debugf("Total CIDRs to program - %d", len(allCIDRs)) // build IPTABLES chain for SNAT of non-VPC outbound traffic and excluded CIDRs var chains []string for i := 0; i <= len(allCIDRs); i++ {