Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

tests: reduce duplicate configurations in hot scheduler tests #8081

Merged
merged 3 commits into from
Apr 17, 2024
Merged

tests: reduce duplicate configurations in hot scheduler tests #8081

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
75e5ca0
tests: reduce duplicate configurations in hot scheduler tests
lhy1024 Apr 17, 2024
d25ee06
add comments
lhy1024 Apr 17, 2024
e31531a
Merge branch 'master' into fix-test19
ti-chi-bot[bot] Apr 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
69 changes: 3 additions & 66 deletions pkg/schedule/schedulers/hot_region_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ func init() {
// TODO: remove this global variable in the future.
// And use a function to create hot schduler for test.
schedulePeerPr = 1.0
// disable denoising in test.
statistics.Denoising = false
statisticsInterval = 0
RegisterScheduler(utils.Write.String(), func(opController *operator.Controller, _ endpoint.ConfigStorage, _ ConfigDecoder, _ ...func(string) error) (Scheduler, error) {
cfg := initHotRegionScheduleConfig()
return newHotWriteScheduler(opController, cfg), nil
Expand Down Expand Up @@ -200,10 +203,8 @@ func checkGCPendingOpInfos(re *require.Assertions, enablePlacementRules bool) {

func TestSplitIfRegionTooHot(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
tc.SetHotRegionCacheHitsThreshold(1)
hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil)
re.NoError(err)
b := &metapb.Buckets{
Expand Down Expand Up @@ -274,9 +275,7 @@ func TestSplitIfRegionTooHot(t *testing.T) {

func TestSplitBucketsBySize(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
cancel, _, tc, oc := prepareSchedulersTest()
tc.SetHotRegionCacheHitsThreshold(1)
tc.SetRegionBucketEnabled(true)
defer cancel()
hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil)
Expand Down Expand Up @@ -327,9 +326,7 @@ func TestSplitBucketsBySize(t *testing.T) {

func TestSplitBucketsByLoad(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
cancel, _, tc, oc := prepareSchedulersTest()
tc.SetHotRegionCacheHitsThreshold(1)
tc.SetRegionBucketEnabled(true)
defer cancel()
hb, err := CreateScheduler(utils.Read.String(), oc, storage.NewStorageWithMemoryBackend(), nil)
Expand Down Expand Up @@ -388,8 +385,6 @@ func TestSplitBucketsByLoad(t *testing.T) {

func TestHotWriteRegionScheduleByteRateOnly(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0
checkHotWriteRegionScheduleByteRateOnly(re, false /* disable placement rules */)
checkHotWriteRegionScheduleByteRateOnly(re, true /* enable placement rules */)
checkHotWriteRegionPlacement(re, true)
Expand All @@ -406,7 +401,6 @@ func checkHotWriteRegionPlacement(re *require.Assertions, enablePlacementRules b
hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil)
re.NoError(err)
hb.(*hotScheduler).conf.SetHistorySampleDuration(0)
tc.SetHotRegionCacheHitsThreshold(0)

tc.AddLabelsStore(1, 2, map[string]string{"zone": "z1", "host": "h1"})
tc.AddLabelsStore(2, 2, map[string]string{"zone": "z1", "host": "h2"})
Expand Down Expand Up @@ -633,12 +627,9 @@ func checkHotWriteRegionScheduleByteRateOnly(re *require.Assertions, enablePlace

func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0
cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.ConfChangeV2))
tc.SetHotRegionCacheHitsThreshold(0)
re.NoError(tc.RuleManager.SetRules([]*placement.Rule{
{
GroupID: placement.DefaultGroupID,
Expand Down Expand Up @@ -853,8 +844,6 @@ func TestHotWriteRegionScheduleWithQuery(t *testing.T) {
}()
cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
statistics.Denoising = false
statisticsInterval = 0

hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil)
re.NoError(err)
Expand All @@ -863,7 +852,6 @@ func TestHotWriteRegionScheduleWithQuery(t *testing.T) {
hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{utils.QueryPriority, utils.BytePriority}
hb.(*hotScheduler).conf.SetHistorySampleDuration(0)

tc.SetHotRegionCacheHitsThreshold(0)
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
tc.AddRegionStore(3, 20)
Expand All @@ -888,8 +876,6 @@ func TestHotWriteRegionScheduleWithQuery(t *testing.T) {

func TestHotWriteRegionScheduleWithKeyRate(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0

cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
Expand All @@ -901,7 +887,6 @@ func TestHotWriteRegionScheduleWithKeyRate(t *testing.T) {
hb.(*hotScheduler).conf.RankFormulaVersion = "v1"
hb.(*hotScheduler).conf.SetHistorySampleDuration(0)

tc.SetHotRegionCacheHitsThreshold(0)
tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0))
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
Expand Down Expand Up @@ -948,8 +933,6 @@ func TestHotWriteRegionScheduleWithKeyRate(t *testing.T) {

func TestHotWriteRegionScheduleUnhealthyStore(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0

cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
Expand All @@ -958,7 +941,6 @@ func TestHotWriteRegionScheduleUnhealthyStore(t *testing.T) {
hb.(*hotScheduler).conf.SetDstToleranceRatio(1)
hb.(*hotScheduler).conf.SetSrcToleranceRatio(1)

tc.SetHotRegionCacheHitsThreshold(0)
tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0))
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
Expand Down Expand Up @@ -996,8 +978,6 @@ func TestHotWriteRegionScheduleUnhealthyStore(t *testing.T) {

func TestHotWriteRegionScheduleCheckHot(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0

cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
Expand All @@ -1006,7 +986,6 @@ func TestHotWriteRegionScheduleCheckHot(t *testing.T) {
hb.(*hotScheduler).conf.SetDstToleranceRatio(1)
hb.(*hotScheduler).conf.SetSrcToleranceRatio(1)

tc.SetHotRegionCacheHitsThreshold(0)
tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0))
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
Expand All @@ -1031,8 +1010,6 @@ func TestHotWriteRegionScheduleCheckHot(t *testing.T) {

func TestHotWriteRegionScheduleWithLeader(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0

cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
Expand All @@ -1041,7 +1018,6 @@ func TestHotWriteRegionScheduleWithLeader(t *testing.T) {
hb.(*hotScheduler).conf.SetHistorySampleDuration(0)
re.NoError(err)

tc.SetHotRegionCacheHitsThreshold(0)
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
tc.AddRegionStore(3, 20)
Expand Down Expand Up @@ -1094,8 +1070,6 @@ func TestHotWriteRegionScheduleWithLeader(t *testing.T) {

func TestHotWriteRegionScheduleWithPendingInfluence(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0
checkHotWriteRegionScheduleWithPendingInfluence(re, 0) // 0: byte rate
checkHotWriteRegionScheduleWithPendingInfluence(re, 1) // 1: key rate
}
Expand All @@ -1114,7 +1088,6 @@ func checkHotWriteRegionScheduleWithPendingInfluence(re *require.Assertions, dim
pendingAmpFactor = old
}()

tc.SetHotRegionCacheHitsThreshold(0)
tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0))
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
Expand Down Expand Up @@ -1189,8 +1162,6 @@ func checkHotWriteRegionScheduleWithPendingInfluence(re *require.Assertions, dim

func TestHotWriteRegionScheduleWithRuleEnabled(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0
cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
tc.SetEnablePlacementRules(true)
Expand All @@ -1199,7 +1170,6 @@ func TestHotWriteRegionScheduleWithRuleEnabled(t *testing.T) {
hb.(*hotScheduler).conf.WriteLeaderPriorities = []string{utils.KeyPriority, utils.BytePriority}
hb.(*hotScheduler).conf.SetHistorySampleDuration(0)

tc.SetHotRegionCacheHitsThreshold(0)
key, err := hex.DecodeString("")
re.NoError(err)
// skip stddev check
Expand Down Expand Up @@ -1282,7 +1252,6 @@ func TestHotReadRegionScheduleByteRateOnly(t *testing.T) {
hb := scheduler.(*hotScheduler)
hb.conf.ReadPriorities = []string{utils.BytePriority, utils.KeyPriority}
hb.conf.SetHistorySampleDuration(0)
tc.SetHotRegionCacheHitsThreshold(0)

// Add stores 1, 2, 3, 4, 5 with region counts 3, 2, 2, 2, 0.
tc.AddRegionStore(1, 3)
Expand Down Expand Up @@ -1396,8 +1365,6 @@ func TestHotReadRegionScheduleByteRateOnly(t *testing.T) {

func TestHotReadRegionScheduleWithQuery(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0

cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
Expand All @@ -1408,7 +1375,6 @@ func TestHotReadRegionScheduleWithQuery(t *testing.T) {
hb.(*hotScheduler).conf.RankFormulaVersion = "v1"
hb.(*hotScheduler).conf.SetHistorySampleDuration(0)

tc.SetHotRegionCacheHitsThreshold(0)
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
tc.AddRegionStore(3, 20)
Expand All @@ -1432,8 +1398,6 @@ func TestHotReadRegionScheduleWithQuery(t *testing.T) {

func TestHotReadRegionScheduleWithKeyRate(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0

cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
Expand All @@ -1445,7 +1409,6 @@ func TestHotReadRegionScheduleWithKeyRate(t *testing.T) {
hb.(*hotScheduler).conf.ReadPriorities = []string{utils.BytePriority, utils.KeyPriority}
hb.(*hotScheduler).conf.SetHistorySampleDuration(0)

tc.SetHotRegionCacheHitsThreshold(0)
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
tc.AddRegionStore(3, 20)
Expand Down Expand Up @@ -1491,8 +1454,6 @@ func TestHotReadRegionScheduleWithKeyRate(t *testing.T) {

func TestHotReadRegionScheduleWithPendingInfluence(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0
checkHotReadRegionScheduleWithPendingInfluence(re, 0) // 0: byte rate
checkHotReadRegionScheduleWithPendingInfluence(re, 1) // 1: key rate
}
Expand All @@ -1515,7 +1476,6 @@ func checkHotReadRegionScheduleWithPendingInfluence(re *require.Assertions, dim
pendingAmpFactor = old
}()

tc.SetHotRegionCacheHitsThreshold(0)
tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0))
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
Expand Down Expand Up @@ -1614,8 +1574,6 @@ func checkHotReadRegionScheduleWithPendingInfluence(re *require.Assertions, dim

func TestHotReadWithEvictLeaderScheduler(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0

cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
Expand All @@ -1625,7 +1583,6 @@ func TestHotReadWithEvictLeaderScheduler(t *testing.T) {
hb.(*hotScheduler).conf.SetDstToleranceRatio(1)
hb.(*hotScheduler).conf.SetStrictPickingStore(false)
hb.(*hotScheduler).conf.ReadPriorities = []string{utils.BytePriority, utils.KeyPriority}
tc.SetHotRegionCacheHitsThreshold(0)
tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0))
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
Expand Down Expand Up @@ -1657,7 +1614,6 @@ func TestHotCacheUpdateCache(t *testing.T) {
re := require.New(t)
cancel, _, tc, _ := prepareSchedulersTest()
defer cancel()
tc.SetHotRegionCacheHitsThreshold(0)

// For read flow
addRegionInfo(tc, utils.Read, []testRegionInfo{
Expand Down Expand Up @@ -1724,7 +1680,6 @@ func TestHotCacheKeyThresholds(t *testing.T) {
{ // only a few regions
cancel, _, tc, _ := prepareSchedulersTest()
defer cancel()
tc.SetHotRegionCacheHitsThreshold(0)
addRegionInfo(tc, utils.Read, []testRegionInfo{
{1, []uint64{1, 2, 3}, 0, 1, 0},
{2, []uint64{1, 2, 3}, 0, 1 * units.KiB, 0},
Expand Down Expand Up @@ -1796,7 +1751,6 @@ func TestHotCacheByteAndKey(t *testing.T) {
re := require.New(t)
cancel, _, tc, _ := prepareSchedulersTest()
defer cancel()
tc.SetHotRegionCacheHitsThreshold(0)
statistics.ThresholdsUpdateInterval = 0
defer func() {
statistics.ThresholdsUpdateInterval = 8 * time.Second
Expand Down Expand Up @@ -2090,8 +2044,6 @@ func TestInfluenceByRWType(t *testing.T) {
defer func() {
schedulePeerPr = originValue
}()
statistics.Denoising = false
statisticsInterval = 0

cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
Expand All @@ -2100,7 +2052,6 @@ func TestInfluenceByRWType(t *testing.T) {
hb.(*hotScheduler).conf.SetDstToleranceRatio(1)
hb.(*hotScheduler).conf.SetSrcToleranceRatio(1)
hb.(*hotScheduler).conf.SetHistorySampleDuration(0)
tc.SetHotRegionCacheHitsThreshold(0)
tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0))
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
Expand Down Expand Up @@ -2214,8 +2165,6 @@ func checkHotReadPeerSchedule(re *require.Assertions, enablePlacementRules bool)

func TestHotScheduleWithPriority(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0

cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
Expand All @@ -2231,7 +2180,6 @@ func TestHotScheduleWithPriority(t *testing.T) {
stddevThreshold = origin
}()

tc.SetHotRegionCacheHitsThreshold(0)
tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0))
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
Expand Down Expand Up @@ -2323,9 +2271,6 @@ func TestHotScheduleWithPriority(t *testing.T) {

func TestHotScheduleWithStddev(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0

cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil)
Expand All @@ -2334,7 +2279,6 @@ func TestHotScheduleWithStddev(t *testing.T) {
hb.(*hotScheduler).conf.SetSrcToleranceRatio(1.0)
hb.(*hotScheduler).conf.RankFormulaVersion = "v1"
tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0))
tc.SetHotRegionCacheHitsThreshold(0)
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
tc.AddRegionStore(3, 20)
Expand Down Expand Up @@ -2384,9 +2328,6 @@ func TestHotScheduleWithStddev(t *testing.T) {

func TestHotWriteLeaderScheduleWithPriority(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0

cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil)
Expand All @@ -2396,7 +2337,6 @@ func TestHotWriteLeaderScheduleWithPriority(t *testing.T) {
hb.(*hotScheduler).conf.SetHistorySampleDuration(0)

tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0))
tc.SetHotRegionCacheHitsThreshold(0)
tc.AddRegionStore(1, 20)
tc.AddRegionStore(2, 20)
tc.AddRegionStore(3, 20)
Expand Down Expand Up @@ -2428,9 +2368,6 @@ func TestHotWriteLeaderScheduleWithPriority(t *testing.T) {

func TestCompatibility(t *testing.T) {
re := require.New(t)
statistics.Denoising = false
statisticsInterval = 0

cancel, _, tc, oc := prepareSchedulersTest()
defer cancel()
hb, err := CreateScheduler(utils.Write.String(), oc, storage.NewStorageWithMemoryBackend(), nil)
Expand Down