-
Notifications
You must be signed in to change notification settings - Fork 4.6k
/
sysctls.go
185 lines (153 loc) · 5.32 KB
/
sysctls.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package model
import (
"fmt"
"strings"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
"k8s.io/kops/util/pkg/distributions"
)
// SysctlBuilder set up our sysctls
type SysctlBuilder struct {
*NodeupModelContext
}
var _ fi.ModelBuilder = &SysctlBuilder{}
// Build is responsible for configuring sysctl settings
func (b *SysctlBuilder) Build(c *fi.ModelBuilderContext) error {
var sysctls []string
// Common settings
{
sysctls = append(sysctls,
"# Kubernetes Settings",
"")
// A higher vm.max_map_count is great for elasticsearch, mongo, or other mmap users
// See https://github.com/kubernetes/kops/issues/1340
sysctls = append(sysctls, "vm.max_map_count = 262144",
"")
// See https://github.com/kubernetes/kubernetes/pull/38001
sysctls = append(sysctls,
"kernel.softlockup_panic = 1",
"kernel.softlockup_all_cpu_backtrace = 1",
"")
// See https://github.com/kubernetes/kops/issues/6342
portRange := b.Cluster.Spec.KubeAPIServer.ServiceNodePortRange
if portRange == "" {
portRange = "30000-32767" // Default kube-apiserver ServiceNodePortRange
}
sysctls = append(sysctls, "net.ipv4.ip_local_reserved_ports = "+portRange,
"")
// See https://github.com/kubernetes/kube-deploy/issues/261
// and https://github.com/kubernetes/kops/issues/10206
sysctls = append(sysctls,
"# Increase the number of connections",
"net.core.somaxconn = 32768",
"",
"# Maximum Socket Receive Buffer",
"net.core.rmem_max = 16777216",
"",
"# Maximum Socket Send Buffer",
"net.core.wmem_max = 16777216",
"",
"# Increase the maximum total buffer-space allocatable",
"net.ipv4.tcp_wmem = 4096 87380 16777216",
"net.ipv4.tcp_rmem = 4096 87380 16777216",
"",
"# Increase the number of outstanding syn requests allowed",
"net.ipv4.tcp_max_syn_backlog = 8096",
"",
"# For persistent HTTP connections",
"net.ipv4.tcp_slow_start_after_idle = 0",
"",
"# Allow to reuse TIME_WAIT sockets for new connections",
"# when it is safe from protocol viewpoint",
"net.ipv4.tcp_tw_reuse = 1",
"",
// We can't change the local_port_range without changing the NodePort range
//"# Allowed local port range",
//"net.ipv4.ip_local_port_range = 10240 65535",
//"",
"# Max number of packets that can be queued on interface input",
"# If kernel is receiving packets faster than can be processed",
"# this queue increases",
"net.core.netdev_max_backlog = 16384",
"",
"# Increase size of file handles and inode cache",
"fs.file-max = 2097152",
"",
"# Max number of inotify instances and watches for a user",
"# Since dockerd runs as a single user, the default instances value of 128 per user is too low",
"# e.g. uses of inotify: nginx ingress controller, kubectl logs -f",
"fs.inotify.max_user_instances = 8192",
"fs.inotify.max_user_watches = 524288",
"# Additional sysctl flags that kubelet expects",
"vm.overcommit_memory = 1",
"kernel.panic = 10",
"kernel.panic_on_oops = 1",
"",
)
}
// Running Flannel on CentOS7 / rhel7 needs custom settings
if b.Cluster.Spec.Networking.Flannel != nil {
proxyMode := b.Cluster.Spec.KubeProxy.ProxyMode
if proxyMode == "" {
proxyMode = "iptables"
}
if proxyMode == "iptables" && (b.Distribution == distributions.DistributionCentos7 || b.Distribution == distributions.DistributionRhel7) {
sysctls = append(sysctls,
"# Flannel settings on CentOS 7",
"# Issue https://github.com/coreos/flannel/issues/902",
"net.bridge.bridge-nf-call-ip6tables=1",
"net.bridge.bridge-nf-call-iptables=1",
"")
}
}
if b.Cluster.Spec.CloudProvider == string(kops.CloudProviderAWS) {
sysctls = append(sysctls,
"# AWS settings",
"",
"# Issue #23395",
"net.ipv4.neigh.default.gc_thresh1=0",
"")
}
if params := b.NodeupConfig.SysctlParameters; len(params) > 0 {
sysctls = append(sysctls,
"# Custom sysctl parameters from instance group spec",
"")
for _, param := range params {
if !strings.ContainsRune(param, '=') {
return fmt.Errorf("Invalid SysctlParameter: expected %q to contain '='", param)
}
sysctls = append(sysctls, param)
}
}
if params := b.Cluster.Spec.SysctlParameters; len(params) > 0 {
sysctls = append(sysctls,
"# Custom sysctl parameters from cluster spec",
"")
for _, param := range params {
if !strings.ContainsRune(param, '=') {
return fmt.Errorf("Invalid SysctlParameter: expected %q to contain '='", param)
}
sysctls = append(sysctls, param)
}
}
c.AddTask(&nodetasks.File{
Path: "/etc/sysctl.d/99-k8s-general.conf",
Contents: fi.NewStringResource(strings.Join(sysctls, "\n")),
Type: nodetasks.FileType_File,
OnChangeExecute: [][]string{{"sysctl", "--system"}},
})
return nil
}