From d5207874e3c02516364993c866b09fad8f05bee3 Mon Sep 17 00:00:00 2001 From: bufferflies <1045931706@qq.com> Date: Thu, 1 Feb 2024 15:21:43 +0800 Subject: [PATCH 1/5] reject modify store limit configuration if store limit version isn't v1 Signed-off-by: bufferflies <1045931706@qq.com> --- server/api/store.go | 16 ++++++++++++- tools/pd-ctl/tests/store/store_test.go | 32 ++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/server/api/store.go b/server/api/store.go index 44e178c23fd..51ac4fb891c 100644 --- a/server/api/store.go +++ b/server/api/store.go @@ -429,6 +429,10 @@ func (h *storeHandler) SetStoreWeight(w http.ResponseWriter, r *http.Request) { // @Router /store/{id}/limit [post] func (h *storeHandler) SetStoreLimit(w http.ResponseWriter, r *http.Request) { rc := getCluster(r) + if version := rc.GetScheduleConfig().StoreLimitVersion; version != storelimit.VersionV1 { + h.rd.JSON(w, http.StatusBadRequest, fmt.Sprintf("current store limit version:%s not support set limit", version)) + return + } vars := mux.Vars(r) storeID, errParse := apiutil.ParseUint64VarsField(vars, "id") if errParse != nil { @@ -529,6 +533,11 @@ func (h *storesHandler) RemoveTombStone(w http.ResponseWriter, r *http.Request) // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /stores/limit [post] func (h *storesHandler) SetAllStoresLimit(w http.ResponseWriter, r *http.Request) { + cfg := h.GetScheduleConfig() + if version := cfg.StoreLimitVersion; version != storelimit.VersionV1 { + h.rd.JSON(w, http.StatusBadRequest, fmt.Sprintf("current store limit version:%s not support get limit", version)) + return + } var input map[string]interface{} if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &input); err != nil { return @@ -609,7 +618,12 @@ func (h *storesHandler) SetAllStoresLimit(w http.ResponseWriter, r *http.Request // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /stores/limit [get] func (h *storesHandler) GetAllStoresLimit(w http.ResponseWriter, r *http.Request) { - limits := h.GetScheduleConfig().StoreLimit + cfg := h.GetScheduleConfig() + if version := cfg.StoreLimitVersion; version != storelimit.VersionV1 { + h.rd.JSON(w, http.StatusBadRequest, fmt.Sprintf("current store limit version:%s not support get limit", version)) + return + } + limits := cfg.StoreLimit includeTombstone := false var err error if includeStr := r.URL.Query().Get("include_tombstone"); includeStr != "" { diff --git a/tools/pd-ctl/tests/store/store_test.go b/tools/pd-ctl/tests/store/store_test.go index c8103414e9c..c7ddc33e9c6 100644 --- a/tools/pd-ctl/tests/store/store_test.go +++ b/tools/pd-ctl/tests/store/store_test.go @@ -38,6 +38,38 @@ import ( "go.etcd.io/etcd/pkg/transport" ) +func TestStoreLimitV2(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cluster, err := pdTests.NewTestCluster(ctx, 1) + re.NoError(err) + err = cluster.RunInitialServers() + re.NoError(err) + cluster.WaitLeader() + pdAddr := cluster.GetConfig().GetClientURL() + cmd := ctl.GetRootCmd() + + leaderServer := cluster.GetLeaderServer() + re.NoError(leaderServer.BootstrapCluster()) + defer cluster.Destroy() + + // store command + args := []string{"-u", pdAddr, "config", "set", "store-limit-version", "v2"} + _, err = tests.ExecuteCommand(cmd, args...) + re.NoError(err) + + args = []string{"-u", pdAddr, "store", "limit"} + output, err := tests.ExecuteCommand(cmd, args...) + re.NoError(err) + re.Contains(string(output), "not support get limit") + + args = []string{"-u", pdAddr, "store", "limit", "1", "10"} + output, err = tests.ExecuteCommand(cmd, args...) + re.NoError(err) + re.Contains(string(output), "not support set limit") +} + func TestStore(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) From f4837489061c6190b50d215cb2077c8e92dd058b Mon Sep 17 00:00:00 2001 From: bufferflies <1045931706@qq.com> Date: Thu, 1 Feb 2024 19:15:03 +0800 Subject: [PATCH 2/5] scale in and out Signed-off-by: bufferflies <1045931706@qq.com> --- tools/pd-simulator/simulator/cases/cases.go | 1 + .../simulator/cases/scale_tikv.go | 89 +++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 tools/pd-simulator/simulator/cases/scale_tikv.go diff --git a/tools/pd-simulator/simulator/cases/cases.go b/tools/pd-simulator/simulator/cases/cases.go index 0a8967a8d86..beb29f630d1 100644 --- a/tools/pd-simulator/simulator/cases/cases.go +++ b/tools/pd-simulator/simulator/cases/cases.go @@ -91,6 +91,7 @@ var CaseMap = map[string]func() *Case{ "redundant-balance-region": newRedundantBalanceRegion, "add-nodes": newAddNodes, "add-nodes-dynamic": newAddNodesDynamic, + "scale-in-out": newScaleInOut, "delete-nodes": newDeleteNodes, "region-split": newRegionSplit, "region-merge": newRegionMerge, diff --git a/tools/pd-simulator/simulator/cases/scale_tikv.go b/tools/pd-simulator/simulator/cases/scale_tikv.go new file mode 100644 index 00000000000..a9acac00cb5 --- /dev/null +++ b/tools/pd-simulator/simulator/cases/scale_tikv.go @@ -0,0 +1,89 @@ +// Copyright 2018 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cases + +import ( + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/tools/pd-simulator/simulator/info" + "github.com/tikv/pd/tools/pd-simulator/simulator/simutil" + "go.uber.org/zap" +) + +func newScaleInOut() *Case { + var simCase Case + + storeNum := simutil.CaseConfigure.StoreNum + regionNum := simutil.CaseConfigure.RegionNum + if storeNum == 0 || regionNum == 0 { + storeNum, regionNum = 6, 4000 + } + + for i := 0; i < storeNum; i++ { + s := &Store{ + ID: IDAllocator.nextID(), + Status: metapb.StoreState_Up, + } + if i%2 == 1 { + s.HasExtraUsedSpace = true + } + simCase.Stores = append(simCase.Stores, s) + } + + for i := 0; i < regionNum; i++ { + peers := []*metapb.Peer{ + {Id: IDAllocator.nextID(), StoreId: uint64(i%storeNum + 1)}, + {Id: IDAllocator.nextID(), StoreId: uint64((i+1)%storeNum + 1)}, + {Id: IDAllocator.nextID(), StoreId: uint64((i+2)%storeNum + 1)}, + } + simCase.Regions = append(simCase.Regions, Region{ + ID: IDAllocator.nextID(), + Peers: peers, + Leader: peers[0], + }) + } + + scaleInTick := int64(regionNum * 3 / storeNum) + addEvent := &AddNodesDescriptor{} + addEvent.Step = func(tick int64) uint64 { + if tick > scaleInTick { + storeNum++ + return uint64(storeNum) + } + return 0 + } + + removeEvent := &DeleteNodesDescriptor{} + removeEvent.Step = func(tick int64) uint64 { + if tick > scaleInTick*2 { + return uint64(storeNum) + } + return 0 + } + + simCase.Events = []EventDescriptor{addEvent, removeEvent} + + // storesLastUpdateTime := make([]int64, storeNum+1) + // storeLastAvailable := make([]uint64, storeNum+1) + + simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + newStore := stats[storeNum-1] + simutil.Logger.Info("current counts", + zap.Uint32("peer-count", newStore.GetRegionCount()), + ) + return false + } + return &simCase +} From e65ea40163387b1f882987e183795d7bdfd42abb Mon Sep 17 00:00:00 2001 From: bufferflies <1045931706@qq.com> Date: Fri, 2 Feb 2024 09:53:48 +0800 Subject: [PATCH 3/5] add case for scale in and out Signed-off-by: bufferflies <1045931706@qq.com> --- tools/pd-simulator/README.md | 11 +++++++++++ .../pd-simulator/simulator/cases/scale_tikv.go | 18 ++++-------------- tools/pd-simulator/simulator/node.go | 1 + 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/tools/pd-simulator/README.md b/tools/pd-simulator/README.md index c47024fc24b..107f6c40f64 100644 --- a/tools/pd-simulator/README.md +++ b/tools/pd-simulator/README.md @@ -43,3 +43,14 @@ Run a specific case with an external PD: ```shell ./pd-simulator -pd="http://127.0.0.1:2379" -case="casename" ``` + +Run with tiup playgroudn : +```shell +tiup playground nightly --host 127.0.0.1 --kv.binpath ./pd-simulator --kv=1 --db=0 --kv.config=./tikv.conf +``` +tikv conf +``` +case-name="redundant-balance-region" +sim-tick-interval="1s" +store-io-per-second=100 +``` \ No newline at end of file diff --git a/tools/pd-simulator/simulator/cases/scale_tikv.go b/tools/pd-simulator/simulator/cases/scale_tikv.go index a9acac00cb5..8dce947379d 100644 --- a/tools/pd-simulator/simulator/cases/scale_tikv.go +++ b/tools/pd-simulator/simulator/cases/scale_tikv.go @@ -19,7 +19,6 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/tools/pd-simulator/simulator/info" "github.com/tikv/pd/tools/pd-simulator/simulator/simutil" - "go.uber.org/zap" ) func newScaleInOut() *Case { @@ -58,31 +57,22 @@ func newScaleInOut() *Case { scaleInTick := int64(regionNum * 3 / storeNum) addEvent := &AddNodesDescriptor{} addEvent.Step = func(tick int64) uint64 { - if tick > scaleInTick { - storeNum++ - return uint64(storeNum) + if tick == scaleInTick { + return uint64(storeNum + 1) } return 0 } removeEvent := &DeleteNodesDescriptor{} removeEvent.Step = func(tick int64) uint64 { - if tick > scaleInTick*2 { - return uint64(storeNum) + if tick == scaleInTick*2 { + return uint64(storeNum + 1) } return 0 } - simCase.Events = []EventDescriptor{addEvent, removeEvent} - // storesLastUpdateTime := make([]int64, storeNum+1) - // storeLastAvailable := make([]uint64, storeNum+1) - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { - newStore := stats[storeNum-1] - simutil.Logger.Info("current counts", - zap.Uint32("peer-count", newStore.GetRegionCount()), - ) return false } return &simCase diff --git a/tools/pd-simulator/simulator/node.go b/tools/pd-simulator/simulator/node.go index 68a10a8638e..8b2e43bc1d7 100644 --- a/tools/pd-simulator/simulator/node.go +++ b/tools/pd-simulator/simulator/node.go @@ -70,6 +70,7 @@ func NewNode(s *cases.Store, pdAddr string, config *SimConfig) (*Node, error) { StoreStats: pdpb.StoreStats{ StoreId: s.ID, Capacity: uint64(config.RaftStore.Capacity), + Available: uint64(config.RaftStore.Capacity), StartTime: uint32(time.Now().Unix()), }, } From b2a676e5aecde7c3b715f04d2833dbcea0caa519 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 12 Jun 2024 16:05:00 +0800 Subject: [PATCH 4/5] fix Signed-off-by: lhy1024 --- tools/pd-simulator/simulator/cases/cases.go | 1 + .../simulator/cases/scale_tikv.go | 36 ++++++++++--------- tools/pd-simulator/simulator/node.go | 1 - 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/tools/pd-simulator/simulator/cases/cases.go b/tools/pd-simulator/simulator/cases/cases.go index c4e2f999978..9d8911c8c24 100644 --- a/tools/pd-simulator/simulator/cases/cases.go +++ b/tools/pd-simulator/simulator/cases/cases.go @@ -89,6 +89,7 @@ var IDAllocator idAllocator var CaseMap = map[string]func(*config.SimConfig) *Case{ "balance-leader": newBalanceLeader, "redundant-balance-region": newRedundantBalanceRegion, + "scale-in-out": newScaleInOut, "region-split": newRegionSplit, "region-merge": newRegionMerge, "hot-read": newHotRead, diff --git a/tools/pd-simulator/simulator/cases/scale_tikv.go b/tools/pd-simulator/simulator/cases/scale_tikv.go index 8dce947379d..525b494677d 100644 --- a/tools/pd-simulator/simulator/cases/scale_tikv.go +++ b/tools/pd-simulator/simulator/cases/scale_tikv.go @@ -1,4 +1,4 @@ -// Copyright 2018 TiKV Project Authors. +// Copyright 2024 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,20 +17,22 @@ package cases import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/tikv/pd/pkg/core" + sc "github.com/tikv/pd/tools/pd-simulator/simulator/config" "github.com/tikv/pd/tools/pd-simulator/simulator/info" "github.com/tikv/pd/tools/pd-simulator/simulator/simutil" ) -func newScaleInOut() *Case { +func newScaleInOut(config *sc.SimConfig) *Case { var simCase Case - storeNum := simutil.CaseConfigure.StoreNum - regionNum := simutil.CaseConfigure.RegionNum - if storeNum == 0 || regionNum == 0 { - storeNum, regionNum = 6, 4000 + totalStore := config.TotalStore + totalRegion := config.TotalRegion + replica := int(config.ServerConfig.Replication.MaxReplicas) + if totalStore == 0 || totalRegion == 0 { + totalStore, totalRegion = 6, 4000 } - for i := 0; i < storeNum; i++ { + for i := 0; i < totalStore; i++ { s := &Store{ ID: IDAllocator.nextID(), Status: metapb.StoreState_Up, @@ -41,11 +43,13 @@ func newScaleInOut() *Case { simCase.Stores = append(simCase.Stores, s) } - for i := 0; i < regionNum; i++ { - peers := []*metapb.Peer{ - {Id: IDAllocator.nextID(), StoreId: uint64(i%storeNum + 1)}, - {Id: IDAllocator.nextID(), StoreId: uint64((i+1)%storeNum + 1)}, - {Id: IDAllocator.nextID(), StoreId: uint64((i+2)%storeNum + 1)}, + for i := 0; i < totalRegion; i++ { + peers := make([]*metapb.Peer, 0, replica) + for j := 0; j < replica; j++ { + peers = append(peers, &metapb.Peer{ + Id: simutil.IDAllocator.NextID(), + StoreId: uint64((i+j)%totalStore + 1), + }) } simCase.Regions = append(simCase.Regions, Region{ ID: IDAllocator.nextID(), @@ -54,11 +58,11 @@ func newScaleInOut() *Case { }) } - scaleInTick := int64(regionNum * 3 / storeNum) + scaleInTick := int64(totalRegion * 3 / totalStore) addEvent := &AddNodesDescriptor{} addEvent.Step = func(tick int64) uint64 { if tick == scaleInTick { - return uint64(storeNum + 1) + return uint64(totalStore + 1) } return 0 } @@ -66,13 +70,13 @@ func newScaleInOut() *Case { removeEvent := &DeleteNodesDescriptor{} removeEvent.Step = func(tick int64) uint64 { if tick == scaleInTick*2 { - return uint64(storeNum + 1) + return uint64(totalStore + 1) } return 0 } simCase.Events = []EventDescriptor{addEvent, removeEvent} - simCase.Checker = func(regions *core.RegionsInfo, stats []info.StoreStats) bool { + simCase.Checker = func(_ *core.RegionsInfo, _ []info.StoreStats) bool { return false } return &simCase diff --git a/tools/pd-simulator/simulator/node.go b/tools/pd-simulator/simulator/node.go index 8a810e85211..fe8dc74a944 100644 --- a/tools/pd-simulator/simulator/node.go +++ b/tools/pd-simulator/simulator/node.go @@ -71,7 +71,6 @@ func NewNode(s *cases.Store, pdAddr string, config *sc.SimConfig) (*Node, error) StoreStats: pdpb.StoreStats{ StoreId: s.ID, Capacity: uint64(config.RaftStore.Capacity), - Available: uint64(config.RaftStore.Capacity), StartTime: uint32(time.Now().Unix()), Available: uint64(config.RaftStore.Capacity), }, From 4148657181c054c40a8c63b85aadfc320330062c Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 27 Jun 2024 23:30:14 +0800 Subject: [PATCH 5/5] fix conflict Signed-off-by: lhy1024 --- tools/pd-simulator/simulator/cases/scale_tikv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pd-simulator/simulator/cases/scale_tikv.go b/tools/pd-simulator/simulator/cases/scale_tikv.go index 525b494677d..96d44513ae7 100644 --- a/tools/pd-simulator/simulator/cases/scale_tikv.go +++ b/tools/pd-simulator/simulator/cases/scale_tikv.go @@ -76,7 +76,7 @@ func newScaleInOut(config *sc.SimConfig) *Case { } simCase.Events = []EventDescriptor{addEvent, removeEvent} - simCase.Checker = func(_ *core.RegionsInfo, _ []info.StoreStats) bool { + simCase.Checker = func([]*metapb.Store, *core.RegionsInfo, []info.StoreStats) bool { return false } return &simCase