-
Notifications
You must be signed in to change notification settings - Fork 5.7k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
*: Support to trigger GC when memory usage is large. #38179
Changes from 19 commits
4b65e3c
d2f00d2
9bfe357
ec3cf48
2506a14
33dfdb0
a2ed5f2
5cb85ee
65fa37c
77870f3
01116aa
73b9678
f47abc0
4764c92
58fa721
035a1e7
7b8a72f
6cb6a26
2ebcd0c
2321b61
0c39f03
fd76ecb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
@@ -0,0 +1,126 @@ | ||||||||||||||||||
// Copyright 2022 PingCAP, Inc. | ||||||||||||||||||
// | ||||||||||||||||||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||||||||||||||||||
// you may not use this file except in compliance with the License. | ||||||||||||||||||
// You may obtain a copy of the License at | ||||||||||||||||||
// | ||||||||||||||||||
// http://www.apache.org/licenses/LICENSE-2.0 | ||||||||||||||||||
// | ||||||||||||||||||
// Unless required by applicable law or agreed to in writing, software | ||||||||||||||||||
// distributed under the License is distributed on an "AS IS" BASIS, | ||||||||||||||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||||||||||||||||||
// See the License for the specific language governing permissions and | ||||||||||||||||||
// limitations under the License. | ||||||||||||||||||
|
||||||||||||||||||
package gctuner | ||||||||||||||||||
|
||||||||||||||||||
import ( | ||||||||||||||||||
"math" | ||||||||||||||||||
"runtime" | ||||||||||||||||||
"runtime/debug" | ||||||||||||||||||
"time" | ||||||||||||||||||
|
||||||||||||||||||
"github.com/pingcap/failpoint" | ||||||||||||||||||
"github.com/pingcap/tidb/util" | ||||||||||||||||||
"github.com/pingcap/tidb/util/memory" | ||||||||||||||||||
atomicutil "go.uber.org/atomic" | ||||||||||||||||||
) | ||||||||||||||||||
|
||||||||||||||||||
// GlobalMemoryLimitTuner only allow one memory limit tuner in one process | ||||||||||||||||||
var GlobalMemoryLimitTuner = &memoryLimitTuner{} | ||||||||||||||||||
|
||||||||||||||||||
// Go runtime trigger GC when hit memory limit which managed via runtime/debug.SetMemoryLimit. | ||||||||||||||||||
// So we can change memory limit dynamically to avoid frequent GC when memory usage is greater than the limit. | ||||||||||||||||||
type memoryLimitTuner struct { | ||||||||||||||||||
finalizer *finalizer | ||||||||||||||||||
isTuning atomicutil.Bool | ||||||||||||||||||
percentage atomicutil.Float64 | ||||||||||||||||||
waitingReset atomicutil.Bool | ||||||||||||||||||
times int // The times that nextGC bigger than MemoryLimit | ||||||||||||||||||
} | ||||||||||||||||||
|
||||||||||||||||||
// tuning check the memory nextGC and judge whether this GC is trigger by memory limit. | ||||||||||||||||||
// Go runtime ensure that it will be called serially. | ||||||||||||||||||
func (t *memoryLimitTuner) tuning() { | ||||||||||||||||||
if !t.isTuning.Load() { | ||||||||||||||||||
return | ||||||||||||||||||
} | ||||||||||||||||||
r := &runtime.MemStats{} | ||||||||||||||||||
runtime.ReadMemStats(r) | ||||||||||||||||||
gogc := util.GetGOGC() | ||||||||||||||||||
ratio := float64(100+gogc) / 100 | ||||||||||||||||||
// If theoretical NextGC(Equivalent to HeapInUse * (100 + GOGC) / 100) is bigger than MemoryLimit twice in a row, | ||||||||||||||||||
// the second GC is caused by MemoryLimit. | ||||||||||||||||||
// All GC is divided into the following three cases: | ||||||||||||||||||
// 1. In normal, HeapInUse * (100 + GOGC) / 100 < MemoryLimit, NextGC = HeapInUse * (100 + GOGC) / 100. | ||||||||||||||||||
// 2. The first time HeapInUse * (100 + GOGC) / 100 >= MemoryLimit, NextGC = MemoryLimit. But this GC is trigger by GOGC. | ||||||||||||||||||
// 3. The second time HeapInUse * (100 + GOGC) / 100 >= MemoryLimit. This GC is trigger by MemoryLimit. | ||||||||||||||||||
// We set MemoryLimit to MaxInt, so the NextGC will be HeapInUse * (100 + GOGC) / 100 again. | ||||||||||||||||||
if float64(r.HeapInuse)*ratio > float64(debug.SetMemoryLimit(-1)) { | ||||||||||||||||||
t.times++ | ||||||||||||||||||
if t.times >= 2 && t.waitingReset.CompareAndSwap(false, true) { | ||||||||||||||||||
t.times = 0 | ||||||||||||||||||
go func() { | ||||||||||||||||||
debug.SetMemoryLimit(math.MaxInt) | ||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. MaxInt or MaxInt64? |
||||||||||||||||||
resetInterval := 1 * time.Minute // Wait 1 minute and set back, to avoid frequent GC | ||||||||||||||||||
failpoint.Inject("testMemoryLimitTuner", func(val failpoint.Value) { | ||||||||||||||||||
if val, ok := val.(bool); val && ok { | ||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why check ok ? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For linter, type assertion must be checked (forcetypeassert) |
||||||||||||||||||
resetInterval = 1 * time.Second | ||||||||||||||||||
} | ||||||||||||||||||
}) | ||||||||||||||||||
time.Sleep(resetInterval) | ||||||||||||||||||
debug.SetMemoryLimit(t.calcMemoryLimit()) | ||||||||||||||||||
for !t.waitingReset.CompareAndSwap(true, false) { | ||||||||||||||||||
continue | ||||||||||||||||||
} | ||||||||||||||||||
}() | ||||||||||||||||||
} | ||||||||||||||||||
} else { | ||||||||||||||||||
t.times = 0 | ||||||||||||||||||
} | ||||||||||||||||||
} | ||||||||||||||||||
|
||||||||||||||||||
func (t *memoryLimitTuner) start() { | ||||||||||||||||||
t.finalizer = newFinalizer(t.tuning) // start tuning | ||||||||||||||||||
} | ||||||||||||||||||
|
||||||||||||||||||
func (t *memoryLimitTuner) stop() { | ||||||||||||||||||
t.finalizer.stop() | ||||||||||||||||||
} | ||||||||||||||||||
|
||||||||||||||||||
// SetPercentage set the percentage for memory limit tuner. | ||||||||||||||||||
func (t *memoryLimitTuner) SetPercentage(percentage float64) { | ||||||||||||||||||
t.percentage.Store(percentage) | ||||||||||||||||||
} | ||||||||||||||||||
|
||||||||||||||||||
// GetPercentage get the percentage from memory limit tuner. | ||||||||||||||||||
func (t *memoryLimitTuner) GetPercentage() float64 { | ||||||||||||||||||
return t.percentage.Load() | ||||||||||||||||||
} | ||||||||||||||||||
|
||||||||||||||||||
// UpdateMemoryLimit updates the memory limit. | ||||||||||||||||||
// This function should be called when `tidb_server_memory_limit` or `tidb_server_memory_limit_gc_trigger` is modified. | ||||||||||||||||||
func (t *memoryLimitTuner) UpdateMemoryLimit() { | ||||||||||||||||||
memoryLimit := t.calcMemoryLimit() | ||||||||||||||||||
if EnableGOGCTuner.Load() { | ||||||||||||||||||
memoryLimit = math.MaxInt64 | ||||||||||||||||||
} | ||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||||||||||
if memoryLimit == math.MaxInt64 { | ||||||||||||||||||
t.isTuning.Store(false) | ||||||||||||||||||
} else { | ||||||||||||||||||
t.isTuning.Store(true) | ||||||||||||||||||
} | ||||||||||||||||||
debug.SetMemoryLimit(memoryLimit) | ||||||||||||||||||
} | ||||||||||||||||||
|
||||||||||||||||||
func (t *memoryLimitTuner) calcMemoryLimit() int64 { | ||||||||||||||||||
memoryLimit := int64(float64(memory.ServerMemoryLimit.Load()) * t.percentage.Load()) // `tidb_server_memory_limit` * `tidb_server_memory_limit_gc_trigger` | ||||||||||||||||||
if memoryLimit == 0 { | ||||||||||||||||||
memoryLimit = math.MaxInt64 | ||||||||||||||||||
} | ||||||||||||||||||
return memoryLimit | ||||||||||||||||||
} | ||||||||||||||||||
|
||||||||||||||||||
func init() { | ||||||||||||||||||
GlobalMemoryLimitTuner.start() | ||||||||||||||||||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,114 @@ | ||
// Copyright 2022 PingCAP, Inc. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package gctuner | ||
|
||
import ( | ||
"math" | ||
"runtime" | ||
"runtime/debug" | ||
"testing" | ||
"time" | ||
|
||
"github.com/pingcap/failpoint" | ||
"github.com/pingcap/tidb/util/memory" | ||
"github.com/stretchr/testify/require" | ||
) | ||
|
||
type mockAllocator struct { | ||
m [][]byte | ||
} | ||
|
||
func (a *mockAllocator) alloc(bytes int) (handle int) { | ||
sli := make([]byte, bytes) | ||
a.m = append(a.m, sli) | ||
return len(a.m) - 1 | ||
} | ||
|
||
func (a *mockAllocator) free(handle int) { | ||
a.m[handle] = nil | ||
} | ||
|
||
func (a *mockAllocator) freeAll() { | ||
a.m = nil | ||
runtime.GC() | ||
} | ||
|
||
func TestGlobalMemoryTuner(t *testing.T) { | ||
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/util/gctuner/testMemoryLimitTuner", "return(true)")) | ||
defer func() { | ||
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/util/gctuner/testMemoryLimitTuner")) | ||
}() | ||
// Close GOGCTuner | ||
gogcTuner := EnableGOGCTuner.Load() | ||
EnableGOGCTuner.Store(false) | ||
defer EnableGOGCTuner.Store(gogcTuner) | ||
|
||
memory.ServerMemoryLimit.Store(1 << 30) // 1GB | ||
GlobalMemoryLimitTuner.SetPercentage(0.8) // 1GB * 80% = 800MB | ||
GlobalMemoryLimitTuner.UpdateMemoryLimit() | ||
require.True(t, GlobalMemoryLimitTuner.isTuning.Load()) | ||
defer func() { | ||
time.Sleep(1 * time.Second) // If test.count > 1, wait tuning finished. | ||
require.True(t, GlobalMemoryLimitTuner.isTuning.Load()) | ||
require.False(t, GlobalMemoryLimitTuner.waitingReset.Load()) | ||
require.Equal(t, GlobalMemoryLimitTuner.times, 0) | ||
}() | ||
|
||
allocator := &mockAllocator{} | ||
defer allocator.freeAll() | ||
r := &runtime.MemStats{} | ||
getNowGCNum := func() uint32 { | ||
runtime.ReadMemStats(r) | ||
return r.NumGC | ||
} | ||
checkNextGCEqualMemoryLimit := func() { | ||
runtime.ReadMemStats(r) | ||
nextGC := r.NextGC | ||
memoryLimit := GlobalMemoryLimitTuner.calcMemoryLimit() | ||
// In golang source, nextGC = memoryLimit - three parts memory. So check 90%~100% here. | ||
require.True(t, nextGC < uint64(memoryLimit)) | ||
require.True(t, nextGC > uint64(memoryLimit)/10*9) | ||
} | ||
|
||
memory600mb := allocator.alloc(600 << 20) | ||
gcNum := getNowGCNum() | ||
|
||
memory210mb := allocator.alloc(210 << 20) | ||
time.Sleep(100 * time.Millisecond) | ||
require.True(t, GlobalMemoryLimitTuner.waitingReset.Load()) | ||
require.True(t, gcNum < getNowGCNum()) | ||
// Test waiting for reset | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we need to check whether |
||
time.Sleep(500 * time.Millisecond) | ||
require.Equal(t, int64(math.MaxInt64), debug.SetMemoryLimit(-1)) | ||
gcNum = getNowGCNum() | ||
memory100mb := allocator.alloc(100 << 20) | ||
time.Sleep(100 * time.Millisecond) | ||
require.Equal(t, gcNum, getNowGCNum()) // No GC | ||
|
||
allocator.free(memory210mb) | ||
allocator.free(memory100mb) | ||
runtime.GC() | ||
// Trigger GC in 80% again | ||
time.Sleep(500 * time.Millisecond) | ||
require.Equal(t, GlobalMemoryLimitTuner.calcMemoryLimit(), debug.SetMemoryLimit(-1)) | ||
time.Sleep(100 * time.Millisecond) | ||
gcNum = getNowGCNum() | ||
checkNextGCEqualMemoryLimit() | ||
memory210mb = allocator.alloc(210 << 20) | ||
time.Sleep(100 * time.Millisecond) | ||
require.True(t, gcNum < getNowGCNum()) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Does this GC trigger by memory_limit? |
||
allocator.free(memory210mb) | ||
allocator.free(memory600mb) | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.