-
Notifications
You must be signed in to change notification settings - Fork 22
/
aws_cni.go
433 lines (430 loc) · 12 KB
/
aws_cni.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
package template
const AwsCNIManifest = `
# Vendored from https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/master/config/v1.6/aws-k8s-cni.yaml
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: aws-cni
spec:
allowPrivilegeEscalation: true
privileged: true
allowedCapabilities:
- 'NET_ADMIN'
fsGroup:
rule: RunAsAny
runAsUser:
rule: RunAsAny
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
hostNetwork: true
hostPorts:
- min: 0
max: 65535
volumes:
- secret
- configMap
- hostPath
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: aws-node
rules:
- apiGroups:
- crd.k8s.amazonaws.com
resources:
- eniconfigs
verbs: ["list", "watch", "get"]
- apiGroups: [""]
resources:
- namespaces
verbs: ["list", "watch", "get"]
- apiGroups: [""]
resources:
- pods
verbs: ["list", "watch", "get"]
- apiGroups: [""]
resources:
- nodes
verbs: ["list", "watch", "get", "update"]
- apiGroups: ["extensions"]
resources:
- '*'
verbs: ["list", "watch"]
- apiGroups: ["policy"]
resources: ["podsecuritypolicies"]
resourceNames: ["aws-cni"]
verbs: ["use", "get", "create"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: aws-node
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: aws-node
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: aws-node
subjects:
- kind: ServiceAccount
name: aws-node
namespace: kube-system
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: aws-node
namespace: kube-system
labels:
k8s-app: aws-node
spec:
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: "10%"
selector:
matchLabels:
k8s-app: aws-node
template:
metadata:
labels:
k8s-app: aws-node
spec:
priorityClassName: system-node-critical
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: "kubernetes.io/os"
operator: In
values:
- linux
- key: "kubernetes.io/arch"
operator: In
values:
- amd64
- arm64
- key: eks.amazonaws.com/compute-type
operator: NotIn
values:
- fargate
serviceAccountName: aws-node
hostNetwork: true
terminationGracePeriodSeconds: 10
tolerations:
- operator: Exists
initContainers:
- image: {{.RegistryDomain}}/giantswarm/aws-cni-init:v{{.AWSCNIVersion}}
imagePullPolicy: Always
name: aws-vpc-cni-init
env:
- name: DISABLE_TCP_EARLY_DEMUX
value: "false"
- name: ENABLE_IPv4
value: "true"
- name: ENABLE_IPv6
value: "false"
- name: ENABLE_NFTABLES
value: "true"
securityContext:
privileged: true
volumeMounts:
- mountPath: /host/opt/cni/bin
name: cni-bin-dir
- image: {{.RegistryDomain}}/giantswarm/alpine:3.17.3
imagePullPolicy: Always
name: setup-sysctl
command:
- ash
- "-c"
- "echo 'net.ipv4.conf.eth0.rp_filter=2' >/host/etc/sysctl.d/99-aws-node.conf"
securityContext:
privileged: true
volumeMounts:
- mountPath: /host/etc/sysctl.d
name: sysctl-d
containers:
- image: {{.RegistryDomain}}/giantswarm/aws-cni:v{{.AWSCNIVersion}}
ports:
- containerPort: 61678
name: metrics
name: aws-node
livenessProbe:
exec:
command:
- /app/grpc-health-probe
- -addr=:50051
- -connect-timeout=5s
- -rpc-timeout=5s
initialDelaySeconds: 60
timeoutSeconds: 10
readinessProbe:
exec:
command:
- /app/grpc-health-probe
- -addr=:50051
- -connect-timeout=5s
- -rpc-timeout=5s
initialDelaySeconds: 1
timeoutSeconds: 10
env:
- name: ADDITIONAL_ENI_TAGS
value: '{{ .AWSCNIAdditionalTags }}'
- name: AWS_VPC_K8S_CNI_LOGLEVEL
value: INFO
- name: AWS_VPC_K8S_PLUGIN_LOG_LEVEL
value: INFO
- name: AWS_VPC_K8S_CNI_LOG_FILE
value: stdout
- name: AWS_VPC_K8S_PLUGIN_LOG_FILE
value: /host/var/log/aws-routed-eni/plugin.log
- name: AWS_VPC_ENI_MTU
value: "9001"
- name: AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER
value: "false"
- name: DISABLE_INTROSPECTION
value: "false"
- name: DISABLE_METRICS
value: "false"
- name: ENABLE_IPv4
value: "true"
- name: ENABLE_IPv6
value: "false"
- name: ENABLE_NFTABLES
value: "true"
- name: POD_SECURITY_GROUP_ENFORCING_MODE
value: standard
## If CNI prefix validation is enabled we remove WARM_IP_TARGET and MINIMUM_IP_TARGET because it will take precedence over WARM_PREFIX_TARGET.
{{- if eq .AWSCNIPrefix false }}
- name: WARM_IP_TARGET
value: "{{ .AWSCNIWarmIPTarget }}"
- name: MINIMUM_IP_TARGET
value: "{{ .AWSCNIMinimumIPTarget }}"
{{- end }}
## Deviation from original manifest - 1
## This config value is important - See here https://github.com/aws/amazon-vpc-cni-k8s/blob/master/README.md#cni-configuration-variables
- name: AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG
value: "true"
## Deviation from original manifest - 2
## setting custom ENI config annotation
- name: ENI_CONFIG_LABEL_DEF
value: "failure-domain.beta.kubernetes.io/zone"
- name: MY_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
## Deviation from original manifest - 3
## disable SNAT as we setup NATGW in the route tables
- name: AWS_VPC_K8S_CNI_EXTERNALSNAT
value: "{{.ExternalSNAT}}"
{{- if eq .ExternalSNAT false }}
## Deviation from original manifest - 4
## If we left this enabled, cross subnet communication doesn't work. Only affects ExternalSNAT=false.
- name: AWS_VPC_K8S_CNI_RANDOMIZESNAT
value: "none"
{{- else }}
## When enabling ExternalSNAT, we need to set this to prng (default).
- name: AWS_VPC_K8S_CNI_RANDOMIZESNAT
value: prng
{{- end }}
## Deviation from original manifest - 5
## Explicit interface naming
- name: AWS_VPC_K8S_CNI_VETHPREFIX
value: eni
{{- if eq .AWSCNIPrefix true }}
## Deviation from original manifest - 6
## If CNI prefix validation is enabled it will improve the speed of allocating IPs on a nitro based node (e.g. m5.2xlarge)
## By setting a annotation on the Cluster CR it can be enabled or disabled.
- name: ENABLE_PREFIX_DELEGATION
value: "true"
- name: "WARM_PREFIX_TARGET"
value: "1"
{{- end }}
resources:
requests:
cpu: 30m
securityContext:
capabilities:
add:
- NET_ADMIN
volumeMounts:
- mountPath: /host/opt/cni/bin
name: cni-bin-dir
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
- mountPath: /host/var/log/aws-routed-eni
name: log-dir
- mountPath: /var/run/aws-node
name: run-dir
- mountPath: /var/run/cri.sock
name: cri
- mountPath: /run/xtables.lock
name: xtables-lock
volumes:
- name: cni-bin-dir
hostPath:
path: /opt/cni/bin
- name: cni-net-dir
hostPath:
path: /etc/cni/net.d
- name: cri
hostPath:
path: /var/run/containerd/containerd.sock
- hostPath:
path: /run/xtables.lock
type: FileOrCreate
name: xtables-lock
- hostPath:
path: /var/log/aws-routed-eni
type: DirectoryOrCreate
name: log-dir
- hostPath:
path: /var/run/aws-node
type: DirectoryOrCreate
name: run-dir
- hostPath:
path: /etc/sysctl.d
type: DirectoryOrCreate
name: sysctl-d
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: eniconfigs.crd.k8s.amazonaws.com
spec:
scope: Cluster
group: crd.k8s.amazonaws.com
preserveUnknownFields: false
versions:
- name: v1alpha1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true
names:
plural: eniconfigs
singular: eniconfig
kind: ENIConfig
---
## AWS CNI restarter, to be removed when AWS CNI is able to detect new Additional CIDRs https://github.com/giantswarm/giantswarm/issues/11077
apiVersion: v1
kind: ServiceAccount
metadata:
name: aws-cni-restarter
namespace: kube-system
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: aws-cni-restarter
namespace: kube-system
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "create", "patch"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["list", "delete"]
- apiGroups: ["policy"]
resources: ["podsecuritypolicies"]
resourceNames: ["aws-cni-restarter"]
verbs: ["use", "get", "create"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: aws-cni-restarter-binding
namespace: kube-system
subjects:
- kind: ServiceAccount
name: aws-cni-restarter
namespace: kube-system
roleRef:
kind: Role
name: aws-cni-restarter
apiGroup: ""
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: aws-cni-restarter
spec:
fsGroup:
rule: RunAsAny
hostNetwork: true
privileged: false
runAsUser:
rule: MustRunAsNonRoot
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
volumes:
- secret
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
labels:
app: aws-cni-restarter
name: aws-cni-restarter
namespace: kube-system
spec:
egress:
- {}
podSelector:
matchLabels:
app: aws-cni-restarter
policyTypes:
- Egress
---
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: aws-cni-restarter
namespace: kube-system
spec:
suspend: true
schedule: "*/5 * * * *"
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 5
failedJobsHistoryLimit: 10
jobTemplate:
spec:
template:
metadata:
labels:
app: aws-cni-restarter
spec:
serviceAccountName: aws-cni-restarter
hostNetwork: true
containers:
- name: aws-cni-restarter
image: {{.RegistryDomain}}/giantswarm/aws-cni-restarter:1.0.2
restartPolicy: OnFailure
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: role
operator: In
values:
- master
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
`