/
instance.go
284 lines (246 loc) · 9.87 KB
/
instance.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
// http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.
package ec2
import (
"fmt"
"strings"
"sync"
"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2/api"
"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/vpc"
"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils"
)
// ec2Instance stores all the information that can be shared across the providers for an instance
type ec2Instance struct {
// lock is to prevent concurrent writes to the fields of the ec2Instance
lock sync.RWMutex
// name is the k8s name of the node
name string
// os is the operating system of the worker node
os string
// instanceId of the worker node
instanceID string
// instanceType is the EC2 instance type
instanceType string
// subnetId is the instance's subnet id
instanceSubnetID string
// instanceSubnetCidrBlock is the cidr block of the instance's subnet
instanceSubnetCidrBlock string
// currentSubnetID can either point to the Subnet ID of the instance or subnet ID from the ENIConfig
currentSubnetID string
// currentSubnetCIDRBlock can either point to the Subnet CIDR block for instance subnet or subnet from ENIConfig
currentSubnetCIDRBlock string
// currentInstanceSecurityGroups can either point to the primary network interface security groups or the security groups in ENIConfig
currentInstanceSecurityGroups []string
// subnetMask is the mask of the subnet CIDR block
subnetMask string
// deviceIndexes is the list of indexes used by the EC2 Instance
deviceIndexes []bool
// primaryENIGroups is the security group used by the primary network interface
primaryENISecurityGroups []string
// primaryENIID is the ID of the primary network interface of the instance
primaryENIID string
// newCustomNetworkingSubnetID is the SubnetID from the ENIConfig
newCustomNetworkingSubnetID string
// newCustomNetworkingSecurityGroups is the security groups from the ENIConfig
newCustomNetworkingSecurityGroups []string
}
// EC2Instance exposes the immutable details of an ec2 instance and common operations on an EC2 Instance
type EC2Instance interface {
LoadDetails(ec2APIHelper api.EC2APIHelper) error
GetHighestUnusedDeviceIndex() (int64, error)
FreeDeviceIndex(index int64)
Name() string
Os() string
Type() string
InstanceID() string
SubnetID() string
SubnetMask() string
SubnetCidrBlock() string
PrimaryNetworkInterfaceID() string
CurrentInstanceSecurityGroups() []string
SetNewCustomNetworkingSpec(subnetID string, securityGroup []string)
UpdateCurrentSubnetAndCidrBlock(helper api.EC2APIHelper) error
}
// NewEC2Instance returns a new EC2 Instance type
func NewEC2Instance(nodeName string, instanceID string, os string) EC2Instance {
return &ec2Instance{
name: nodeName,
os: os,
instanceID: instanceID,
}
}
// LoadDetails loads the instance details by making an EC2 API call
func (i *ec2Instance) LoadDetails(ec2APIHelper api.EC2APIHelper) error {
i.lock.Lock()
defer i.lock.Unlock()
instance, err := ec2APIHelper.GetInstanceDetails(&i.instanceID)
if err != nil {
return err
}
if instance == nil || instance.SubnetId == nil {
return fmt.Errorf("failed to find instance %s details from EC2 API", i.instanceID)
}
// Set instance subnet and cidr during node initialization
i.instanceSubnetID = *instance.SubnetId
instanceSubnet, err := ec2APIHelper.GetSubnet(&i.instanceSubnetID)
if err != nil {
return err
}
if instanceSubnet == nil || instanceSubnet.CidrBlock == nil {
return fmt.Errorf("failed to find subnet or CIDR block for subnet %s for instance %s",
i.instanceSubnetID, i.instanceID)
}
i.instanceSubnetCidrBlock = *instanceSubnet.CidrBlock
i.subnetMask = strings.Split(i.instanceSubnetCidrBlock, "/")[1]
i.instanceType = *instance.InstanceType
limits, ok := vpc.Limits[i.instanceType]
if !ok {
return fmt.Errorf("unsupported instance type, couldn't find ENI Limit for instance %s, error: %w", i.instanceType, utils.ErrNotFound)
}
defaultCardIdx := limits.DefaultNetworkCardIndex
var defaultNetworkCardLimit int64
for _, card := range limits.NetworkCards {
if card.NetworkCardIndex == int64(defaultCardIdx) {
defaultNetworkCardLimit = card.MaximumNetworkInterfaces
break
}
}
if defaultNetworkCardLimit == 0 {
return fmt.Errorf("didn't find valid network card with max interface limit from limit file for instance type %s", i.instanceType)
}
// currently CNI and this controller both only support single network card
// we want to make sure to use the smaller number between instance max supported interfaces and the default card max supported interfaces
maxInterfaces := utils.Minimum(int64(limits.Interface), defaultNetworkCardLimit)
i.deviceIndexes = make([]bool, int(maxInterfaces))
for _, nwInterface := range instance.NetworkInterfaces {
index := nwInterface.Attachment.DeviceIndex
i.deviceIndexes[*index] = true
// Load the Security group of the primary network interface
if i.primaryENISecurityGroups == nil && (nwInterface.PrivateIpAddress != nil && instance.PrivateIpAddress != nil && *nwInterface.PrivateIpAddress == *instance.PrivateIpAddress) {
i.primaryENIID = *nwInterface.NetworkInterfaceId
// TODO: Group can change, should be refreshed each time we want to use this
for _, group := range nwInterface.Groups {
i.primaryENISecurityGroups = append(i.primaryENISecurityGroups, *group.GroupId)
}
}
}
return i.updateCurrentSubnetAndCidrBlock(ec2APIHelper)
}
// Os returns the os of the instance
func (i *ec2Instance) Os() string {
return i.os
}
// InstanceId returns the instance id of the instance
func (i *ec2Instance) InstanceID() string {
return i.instanceID
}
// SubnetId returns the subnet id of the instance
func (i *ec2Instance) SubnetID() string {
i.lock.RLock()
defer i.lock.RUnlock()
return i.currentSubnetID
}
// SubnetCidrBlock returns the subnet cidr block of the instance
func (i *ec2Instance) SubnetCidrBlock() string {
i.lock.RLock()
defer i.lock.RUnlock()
return i.currentSubnetCIDRBlock
}
// Name returns the name of the node
func (i *ec2Instance) Name() string {
return i.name
}
// Type returns the instance type of the node
func (i *ec2Instance) Type() string {
return i.instanceType
}
func (i *ec2Instance) PrimaryNetworkInterfaceID() string {
return i.primaryENIID
}
// CurrentInstanceSecurityGroups returns the current instance security groups
// (primary network interface SG or SG specified in the ENIConfig)
func (i *ec2Instance) CurrentInstanceSecurityGroups() []string {
i.lock.RLock()
defer i.lock.RUnlock()
return i.currentInstanceSecurityGroups
}
// GetHighestUnusedDeviceIndex assigns a free device index from the end of the list since IPAMD assigns indexes from
// the beginning of the list
func (i *ec2Instance) GetHighestUnusedDeviceIndex() (int64, error) {
i.lock.Lock()
defer i.lock.Unlock()
for index := len(i.deviceIndexes) - 1; index >= 0; index-- {
if i.deviceIndexes[index] == false {
i.deviceIndexes[index] = true
return int64(index), nil
}
}
return 0, fmt.Errorf("no free device index found")
}
// FreeDeviceIndex frees a device index from the list of managed index
func (i *ec2Instance) FreeDeviceIndex(index int64) {
i.lock.Lock()
defer i.lock.Unlock()
i.deviceIndexes[index] = false
}
func (i *ec2Instance) SubnetMask() string {
i.lock.Lock()
defer i.lock.Unlock()
return i.subnetMask
}
// SetNewCustomNetworkingSpec updates the subnet ID and subnet CIDR block for the instance
func (i *ec2Instance) SetNewCustomNetworkingSpec(subnet string, securityGroups []string) {
i.lock.Lock()
defer i.lock.Unlock()
i.newCustomNetworkingSubnetID = subnet
i.newCustomNetworkingSecurityGroups = securityGroups
}
// UpdateCurrentSubnetAndCidrBlock updates the subnet details under a write lock
func (i *ec2Instance) UpdateCurrentSubnetAndCidrBlock(ec2APIHelper api.EC2APIHelper) error {
i.lock.Lock()
defer i.lock.Unlock()
return i.updateCurrentSubnetAndCidrBlock(ec2APIHelper)
}
// updateCurrentSubnetAndCidrBlock updates subnet details and security group if the node is
// using custom networking
func (i *ec2Instance) updateCurrentSubnetAndCidrBlock(ec2APIHelper api.EC2APIHelper) error {
// Custom networking is being used on node, point the current subnet ID, CIDR block and
// instance security group to the one's present in the Custom networking spec
if i.newCustomNetworkingSubnetID != "" {
if i.newCustomNetworkingSecurityGroups != nil && len(i.newCustomNetworkingSecurityGroups) > 0 {
i.currentInstanceSecurityGroups = i.newCustomNetworkingSecurityGroups
} else {
// when security groups are not specified in ENIConfig, use the primary network interface SG as per custom networking documentation
i.currentInstanceSecurityGroups = i.primaryENISecurityGroups
}
// Only get the subnet CIDR block again if the subnet ID has changed
if i.newCustomNetworkingSubnetID != i.currentSubnetID {
customSubnet, err := ec2APIHelper.GetSubnet(&i.newCustomNetworkingSubnetID)
if err != nil {
return err
}
if customSubnet == nil || customSubnet.CidrBlock == nil {
return fmt.Errorf("failed to find subnet %s", i.newCustomNetworkingSubnetID)
}
i.currentSubnetID = i.newCustomNetworkingSubnetID
i.currentSubnetCIDRBlock = *customSubnet.CidrBlock
}
} else {
// Custom networking in not being used, point to the primary network interface security group and
// subnet details
i.currentSubnetID = i.instanceSubnetID
i.currentSubnetCIDRBlock = i.instanceSubnetCidrBlock
i.currentInstanceSecurityGroups = i.primaryENISecurityGroups
}
return nil
}