Skip to content

Commit 9888ef8

Browse files
obeattieOliver Beattie
obeattie
authored and
Oliver Beattie
committed
[PLAT-713] Allow the CFS period to be tuned for containers
1 parent 5ba1472 commit 9888ef8

File tree

5 files changed

+134
-51
lines changed

5 files changed

+134
-51
lines changed

‎pkg/api/v1/types.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3537,6 +3537,13 @@ const (
35373537
// NVIDIA GPU, in devices. Alpha, might change: although fractional and allowing values >1, only one whole device per node is assigned.
35383538
ResourceNvidiaGPU ResourceName = "alpha.kubernetes.io/nvidia-gpu"
35393539
// Number of Pods that may be running on this Node: see ResourcePods
3540+
3541+
// -- Monzo-specific
3542+
3543+
// CPU throttling period, in microseconds.
3544+
// NOTE: Only set this value yourself if you know very well how to tune the
3545+
// Linux CFS scheduler, or in consultation with the Platform team.
3546+
ResourceCPUPeriodUsec ResourceName = "monzo.com/cpu-period"
35403547
)
35413548

35423549
const (

‎pkg/kubelet/cm/helpers_linux.go

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,13 @@ const (
3737
MilliCPUToCPU = 1000
3838

3939
// 100000 is equivalent to 100ms
40-
QuotaPeriod = 100000
41-
MinQuotaPeriod = 1000
40+
DefaultQuotaPeriod int64 = 100000
41+
MinQuotaPeriod int64 = 1000
4242
)
4343

44-
// MilliCPUToQuota converts milliCPU to CFS quota and period values.
45-
func MilliCPUToQuota(milliCPU int64) (quota int64, period int64) {
44+
// MilliCPUToQuota takes milliCPU (along with a CFS period, in usec) and returns
45+
// a CFS quota value
46+
func MilliCPUToQuota(milliCPU, period int64) (quota int64) {
4647
// CFS quota is measured in two values:
4748
// - cfs_period_us=100ms (the amount of time to measure usage across)
4849
// - cfs_quota=20ms (the amount of cpu time allowed to be used across a period)
@@ -53,11 +54,8 @@ func MilliCPUToQuota(milliCPU int64) (quota int64, period int64) {
5354
return
5455
}
5556

56-
// we set the period to 100ms by default
57-
period = QuotaPeriod
58-
5957
// we then convert your milliCPU to a value normalized over a period
60-
quota = (milliCPU * QuotaPeriod) / MilliCPUToCPU
58+
quota = (milliCPU * period) / MilliCPUToCPU
6159

6260
// quota needs to be a minimum of 1ms.
6361
if quota < MinQuotaPeriod {
@@ -93,20 +91,24 @@ func ResourceConfigForPod(pod *v1.Pod) *ResourceConfig {
9391

9492
cpuRequests := int64(0)
9593
cpuLimits := int64(0)
94+
cpuPeriod := DefaultQuotaPeriod
9695
memoryLimits := int64(0)
9796
if request, found := reqs[v1.ResourceCPU]; found {
9897
cpuRequests = request.MilliValue()
9998
}
10099
if limit, found := limits[v1.ResourceCPU]; found {
101100
cpuLimits = limit.MilliValue()
102101
}
102+
if limit, found := limits[v1.ResourceCPUPeriodUsec]; found {
103+
cpuPeriod = limit.Value()
104+
}
103105
if limit, found := limits[v1.ResourceMemory]; found {
104106
memoryLimits = limit.Value()
105107
}
106108

107109
// convert to CFS values
108110
cpuShares := MilliCPUToShares(cpuRequests)
109-
cpuQuota, cpuPeriod := MilliCPUToQuota(cpuLimits)
111+
cpuQuota := MilliCPUToQuota(cpuLimits, cpuPeriod)
110112

111113
// track if limits were applied for each resource.
112114
memoryLimitsDeclared := true

‎pkg/kubelet/cm/helpers_linux_test.go

Lines changed: 104 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,14 @@ func TestResourceConfigForPod(t *testing.T) {
5353
memoryQuantity := resource.MustParse("200Mi")
5454
burstableMemory := memoryQuantity.Value()
5555
burstablePartialShares := MilliCPUToShares(200)
56-
burstableQuota, burstablePeriod := MilliCPUToQuota(200)
56+
burstablePeriod := DefaultQuotaPeriod
57+
burstableQuota := MilliCPUToQuota(200, burstablePeriod)
5758
guaranteedShares := MilliCPUToShares(100)
58-
guaranteedQuota, guaranteedPeriod := MilliCPUToQuota(100)
59+
guaranteedPeriod := int64(10000)
60+
guaranteedQuota := MilliCPUToQuota(100, guaranteedPeriod)
5961
memoryQuantity = resource.MustParse("100Mi")
6062
guaranteedMemory := memoryQuantity.Value()
63+
6164
testCases := map[string]struct {
6265
pod *v1.Pod
6366
expected *ResourceConfig
@@ -67,19 +70,30 @@ func TestResourceConfigForPod(t *testing.T) {
6770
Spec: v1.PodSpec{
6871
Containers: []v1.Container{
6972
{
70-
Resources: getResourceRequirements(getResourceList("", ""), getResourceList("", "")),
73+
Resources: v1.ResourceRequirements{
74+
Requests: v1.ResourceList{},
75+
Limits: v1.ResourceList{},
76+
},
7177
},
7278
},
7379
},
7480
},
75-
expected: &ResourceConfig{CpuShares: &minShares},
81+
expected: &ResourceConfig{
82+
CpuShares: &minShares,
83+
},
7684
},
7785
"burstable-no-limits": {
7886
pod: &v1.Pod{
7987
Spec: v1.PodSpec{
8088
Containers: []v1.Container{
8189
{
82-
Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("", "")),
90+
Resources: v1.ResourceRequirements{
91+
Requests: v1.ResourceList{
92+
v1.ResourceCPU: resource.MustParse("100m"),
93+
v1.ResourceMemory: resource.MustParse("100Mi"),
94+
},
95+
Limits: v1.ResourceList{},
96+
},
8397
},
8498
},
8599
},
@@ -91,39 +105,86 @@ func TestResourceConfigForPod(t *testing.T) {
91105
Spec: v1.PodSpec{
92106
Containers: []v1.Container{
93107
{
94-
Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")),
108+
Resources: v1.ResourceRequirements{
109+
Requests: v1.ResourceList{
110+
v1.ResourceCPU: resource.MustParse("100m"),
111+
v1.ResourceMemory: resource.MustParse("100Mi"),
112+
},
113+
Limits: v1.ResourceList{
114+
v1.ResourceCPU: resource.MustParse("200m"),
115+
v1.ResourceMemory: resource.MustParse("200Mi"),
116+
},
117+
},
95118
},
96119
},
97120
},
98121
},
99-
expected: &ResourceConfig{CpuShares: &burstableShares, CpuQuota: &burstableQuota, CpuPeriod: &burstablePeriod, Memory: &burstableMemory},
122+
expected: &ResourceConfig{
123+
CpuShares: &burstableShares,
124+
CpuQuota: &burstableQuota,
125+
CpuPeriod: &burstablePeriod,
126+
Memory: &burstableMemory,
127+
},
100128
},
101129
"burstable-partial-limits": {
102130
pod: &v1.Pod{
103131
Spec: v1.PodSpec{
104132
Containers: []v1.Container{
105133
{
106-
Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")),
134+
Resources: v1.ResourceRequirements{
135+
Requests: v1.ResourceList{
136+
v1.ResourceCPU: resource.MustParse("100m"),
137+
v1.ResourceMemory: resource.MustParse("100Mi"),
138+
},
139+
Limits: v1.ResourceList{
140+
v1.ResourceCPU: resource.MustParse("200m"),
141+
v1.ResourceMemory: resource.MustParse("200Mi"),
142+
},
143+
},
107144
},
108145
{
109-
Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("", "")),
146+
Resources: v1.ResourceRequirements{
147+
Requests: v1.ResourceList{
148+
v1.ResourceCPU: resource.MustParse("100m"),
149+
v1.ResourceMemory: resource.MustParse("100Mi"),
150+
},
151+
Limits: v1.ResourceList{},
152+
},
110153
},
111154
},
112155
},
113156
},
114-
expected: &ResourceConfig{CpuShares: &burstablePartialShares},
157+
expected: &ResourceConfig{
158+
CpuShares: &burstablePartialShares,
159+
},
115160
},
116161
"guaranteed": {
117162
pod: &v1.Pod{
118163
Spec: v1.PodSpec{
119164
Containers: []v1.Container{
120165
{
121-
Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")),
166+
Resources: v1.ResourceRequirements{
167+
Requests: v1.ResourceList{
168+
v1.ResourceCPU: resource.MustParse("100m"),
169+
v1.ResourceMemory: resource.MustParse("100Mi"),
170+
v1.ResourceCPUPeriodUsec: resource.MustParse("10000"),
171+
},
172+
Limits: v1.ResourceList{
173+
v1.ResourceCPU: resource.MustParse("100m"),
174+
v1.ResourceMemory: resource.MustParse("100Mi"),
175+
v1.ResourceCPUPeriodUsec: resource.MustParse("10000"),
176+
},
177+
},
122178
},
123179
},
124180
},
125181
},
126-
expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &guaranteedPeriod, Memory: &guaranteedMemory},
182+
expected: &ResourceConfig{
183+
CpuShares: &guaranteedShares,
184+
CpuQuota: &guaranteedQuota,
185+
CpuPeriod: &guaranteedPeriod,
186+
Memory: &guaranteedMemory,
187+
},
127188
},
128189
}
129190
for testName, testCase := range testCases {
@@ -145,55 +206,65 @@ func TestResourceConfigForPod(t *testing.T) {
145206

146207
func TestMilliCPUToQuota(t *testing.T) {
147208
testCases := []struct {
148-
input int64
149-
quota int64
209+
cpu int64
150210
period int64
211+
quota int64
151212
}{
152213
{
153-
input: int64(0),
214+
cpu: int64(0),
215+
period: int64(100000),
154216
quota: int64(0),
155-
period: int64(0),
156217
},
157218
{
158-
input: int64(5),
159-
quota: int64(1000),
219+
cpu: int64(5),
160220
period: int64(100000),
221+
quota: int64(1000),
161222
},
162223
{
163-
input: int64(9),
164-
quota: int64(1000),
224+
cpu: int64(9),
165225
period: int64(100000),
226+
quota: int64(1000),
166227
},
167228
{
168-
input: int64(10),
169-
quota: int64(1000),
229+
cpu: int64(10),
170230
period: int64(100000),
231+
quota: int64(1000),
171232
},
172233
{
173-
input: int64(200),
174-
quota: int64(20000),
234+
cpu: int64(200),
175235
period: int64(100000),
236+
quota: int64(20000),
176237
},
177238
{
178-
input: int64(500),
179-
quota: int64(50000),
239+
cpu: int64(500),
180240
period: int64(100000),
241+
quota: int64(50000),
181242
},
182243
{
183-
input: int64(1000),
184-
quota: int64(100000),
244+
cpu: int64(1000),
185245
period: int64(100000),
246+
quota: int64(100000),
186247
},
187248
{
188-
input: int64(1500),
189-
quota: int64(150000),
249+
cpu: int64(1500),
190250
period: int64(100000),
251+
quota: int64(150000),
252+
},
253+
{
254+
cpu: int64(1500),
255+
period: int64(10000),
256+
quota: int64(15000),
257+
},
258+
{
259+
cpu: int64(250),
260+
period: int64(5000),
261+
quota: int64(1250),
191262
},
192263
}
193264
for _, testCase := range testCases {
194-
quota, period := MilliCPUToQuota(testCase.input)
195-
if quota != testCase.quota || period != testCase.period {
196-
t.Errorf("Input %v, expected quota %v period %v, but got quota %v period %v", testCase.input, testCase.quota, testCase.period, quota, period)
265+
quota := MilliCPUToQuota(testCase.cpu, testCase.period)
266+
if quota != testCase.quota {
267+
t.Errorf("Input (cpu=%d, period=%d), expected quota=%d but got quota=%d", testCase.cpu, testCase.period, testCase.quota, quota)
197268
}
198269
}
199270
}

‎pkg/kubelet/kuberuntime/helpers.go

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ const (
3636
milliCPUToCPU = 1000
3737

3838
// 100000 is equivalent to 100ms
39-
quotaPeriod = 100 * minQuotaPeriod
40-
minQuotaPeriod = 1000
39+
defaultQuotaPeriod int64 = 100000
40+
minQuotaPeriod int64 = 1000
4141
)
4242

4343
var (
@@ -176,22 +176,21 @@ func milliCPUToShares(milliCPU int64) int64 {
176176
return shares
177177
}
178178

179-
// milliCPUToQuota converts milliCPU to CFS quota and period values
180-
func milliCPUToQuota(milliCPU int64) (quota int64, period int64) {
179+
// milliCPUToQuota takes milliCPU (along with a CFS period, in usec) and returns
180+
// a CFS quota value
181+
func milliCPUToQuota(milliCPU, period int64) (quota int64) {
181182
// CFS quota is measured in two values:
182183
// - cfs_period_us=100ms (the amount of time to measure usage across)
183184
// - cfs_quota=20ms (the amount of cpu time allowed to be used across a period)
184185
// so in the above example, you are limited to 20% of a single CPU
185186
// for multi-cpu environments, you just scale equivalent amounts
187+
186188
if milliCPU == 0 {
187189
return
188190
}
189191

190-
// we set the period to 100ms by default
191-
period = quotaPeriod
192-
193192
// we then convert your milliCPU to a value normalized over a period
194-
quota = (milliCPU * quotaPeriod) / milliCPUToCPU
193+
quota = (milliCPU * period) / milliCPUToCPU
195194

196195
// quota needs to be a minimum of 1ms.
197196
if quota < minQuotaPeriod {

‎pkg/kubelet/kuberuntime/kuberuntime_container.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,9 +246,13 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerConfig(container *v1.C
246246
lc.Resources.OomScoreAdj = oomScoreAdj
247247

248248
if m.cpuCFSQuota {
249+
cpuPeriod := defaultQuotaPeriod
250+
if period, found := container.Resources.Limits[v1.ResourceCPUPeriodUsec]; found {
251+
cpuPeriod = period.Value()
252+
}
249253
// if cpuLimit.Amount is nil, then the appropriate default value is returned
250254
// to allow full usage of cpu resource.
251-
cpuQuota, cpuPeriod := milliCPUToQuota(cpuLimit.MilliValue())
255+
cpuQuota := milliCPUToQuota(cpuLimit.MilliValue(), cpuPeriod)
252256
lc.Resources.CpuQuota = cpuQuota
253257
lc.Resources.CpuPeriod = cpuPeriod
254258
}

0 commit comments

Comments
 (0)