From b7c9e49a01d54f579dc0124d8171dcb5ece62959 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 15 Jun 2023 18:53:53 +0800 Subject: [PATCH 1/8] fix client Signed-off-by: lhy1024 --- client/client.go | 19 +++++++++++++------ client/tso_client.go | 4 +--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/client/client.go b/client/client.go index c2f673c5da0..eb61d0a3631 100644 --- a/client/client.go +++ b/client/client.go @@ -277,6 +277,14 @@ type serviceModeKeeper struct { tsoSvcDiscovery ServiceDiscovery } +func (k *serviceModeKeeper) SetKeyspaceID(keyspaceID uint32) { + k.Lock() + defer k.Unlock() + if k.serviceMode == pdpb.ServiceMode_API_SVC_MODE { + k.tsoSvcDiscovery.SetKeyspaceID(keyspaceID) + } +} + func (k *serviceModeKeeper) close() { k.Lock() defer k.Unlock() @@ -471,9 +479,6 @@ func newClientWithKeyspaceName( ctx context.Context, keyspaceName string, svrAddrs []string, security SecurityOption, opts ...ClientOption, ) (Client, error) { - log.Info("[pd] create pd client with endpoints and keyspace", - zap.Strings("pd-address", svrAddrs), zap.String("keyspace-name", keyspaceName)) - tlsCfg := &tlsutil.TLSConfig{ CAPath: security.CAPath, CertPath: security.CertPath, @@ -511,7 +516,9 @@ func newClientWithKeyspaceName( return nil, err } c.pdSvcDiscovery.SetKeyspaceID(c.keyspaceID) - + c.serviceModeKeeper.SetKeyspaceID(c.keyspaceID) + log.Info("[pd] create pd client with endpoints and keyspace", + zap.Strings("pd-address", svrAddrs), zap.String("keyspace-name", keyspaceName), zap.Uint32("keyspace-id", c.keyspaceID)) return c, nil } @@ -593,7 +600,7 @@ func (c *client) setServiceMode(newMode pdpb.ServiceMode) { ) switch newMode { case pdpb.ServiceMode_PD_SVC_MODE: - newTSOCli = newTSOClient(c.ctx, c.option, c.keyspaceID, + newTSOCli = newTSOClient(c.ctx, c.option, c.pdSvcDiscovery, &pdTSOStreamBuilderFactory{}) case pdpb.ServiceMode_API_SVC_MODE: newTSOSvcDiscovery = newTSOServiceDiscovery( @@ -601,7 +608,7 @@ func (c *client) setServiceMode(newMode pdpb.ServiceMode) { c.GetClusterID(c.ctx), c.keyspaceID, c.tlsCfg, c.option) // At this point, the keyspace group isn't known yet. Starts from the default keyspace group, // and will be updated later. - newTSOCli = newTSOClient(c.ctx, c.option, c.keyspaceID, + newTSOCli = newTSOClient(c.ctx, c.option, newTSOSvcDiscovery, &tsoTSOStreamBuilderFactory{}) if err := newTSOSvcDiscovery.Init(); err != nil { log.Error("[pd] failed to initialize tso service discovery. keep the current service mode", diff --git a/client/tso_client.go b/client/tso_client.go index c326e3e7160..d4dfaa03a91 100644 --- a/client/tso_client.go +++ b/client/tso_client.go @@ -70,7 +70,6 @@ type tsoClient struct { wg sync.WaitGroup option *option - keyspaceID uint32 svcDiscovery ServiceDiscovery tsoStreamBuilderFactory // tsoAllocators defines the mapping {dc-location -> TSO allocator leader URL} @@ -94,7 +93,7 @@ type tsoClient struct { // newTSOClient returns a new TSO client. func newTSOClient( - ctx context.Context, option *option, keyspaceID uint32, + ctx context.Context, option *option, svcDiscovery ServiceDiscovery, factory tsoStreamBuilderFactory, ) *tsoClient { ctx, cancel := context.WithCancel(ctx) @@ -102,7 +101,6 @@ func newTSOClient( ctx: ctx, cancel: cancel, option: option, - keyspaceID: keyspaceID, svcDiscovery: svcDiscovery, tsoStreamBuilderFactory: factory, checkTSDeadlineCh: make(chan struct{}), From b6f83d88c846e6ae5b422ad01b5d2a7517a459b1 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 15 Jun 2023 20:35:29 +0800 Subject: [PATCH 2/8] add test Signed-off-by: lhy1024 --- pkg/tso/keyspace_group_manager.go | 6 ++ .../mcs/tso/keyspace_group_manager_test.go | 78 +++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index 3159518387f..d962cfd0306 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -829,6 +829,12 @@ func (kgm *KeyspaceGroupManager) checkTSOSplit( return err } if tsoutil.CompareTimestamp(&splitSourceTSO, &splitTSO) <= 0 { + log.Debug("the split source TSO is not greater than the newly split TSO", + zap.Int64("split-source-tso-physical", splitSourceTSO.Physical), + zap.Int64("split-source-tso-logical", splitSourceTSO.Logical), + zap.Int64("split-tso-physical", splitTSO.Physical), + zap.Int64("split-tso-logical", splitTSO.Logical), + ) return nil } // If the split source TSO is greater than the newly split TSO, we need to update the split diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index 8849f7a4ab5..a91e18f1935 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -33,9 +33,11 @@ import ( "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/storage/endpoint" tsopkg "github.com/tikv/pd/pkg/tso" + "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/server/apiv2/handlers" + "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" "github.com/tikv/pd/tests/integrations/mcs" handlersutil "github.com/tikv/pd/tests/server/apiv2/handlers" @@ -465,3 +467,79 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestTSOKeyspaceGroupMembers() { re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) } + +func TestTwiceSplitKeyspaceGroup(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + + // Init api server config but not start. + tc, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, serverName string) { + conf.Keyspace.PreAlloc = []string{ + "keyspace_a", "keyspace_b", + } + }) + re.NoError(err) + pdAddr := tc.GetConfig().GetClientURL() + + // Start pd client and wait pd server start. + done := make(chan pd.Client) + go func() { + apiCtx := pd.NewAPIContextV2("keyspace_b") // its keyspace id is 2. + cli, err := pd.NewClientWithAPIContext(ctx, apiCtx, []string{pdAddr}, pd.SecurityOption{}) + re.NoError(err) + done <- cli + }() + + // Start api server and tso server. + err = tc.RunInitialServers() + re.NoError(err) + defer tc.Destroy() + tsoServer, tsoServerCleanup1, err := tests.StartSingleTSOTestServer(ctx, re, pdAddr, tempurl.Alloc()) + defer tsoServerCleanup1() + re.NoError(err) + _, tsoServerCleanup2, err := tests.StartSingleTSOTestServer(ctx, re, pdAddr, tempurl.Alloc()) + defer tsoServerCleanup2() + re.NoError(err) + time.Sleep(2 * time.Second) + tc.WaitLeader() + leaderServer := tc.GetServer(tc.GetLeader()) + re.NoError(leaderServer.BootstrapCluster()) + + // First split keyspace group 0 to 1 with keyspace 2. + kgm := leaderServer.GetServer().GetKeyspaceGroupManager() + re.NotNil(kgm) + testutil.Eventually(re, func() bool { + err = kgm.SplitKeyspaceGroupByID(0, 1, []uint32{2}) + return err == nil + }) + + // Set the TSO of the keyspace group 0 to a large value to make test stable. + cli := <-done + defer cli.Close() + physical, logical, err := cli.GetTS(ctx) + physical += time.Hour.Milliseconds() + tsoServer.GetHandler().ResetTS(tsoutil.GenerateTS(&pdpb.Timestamp{ + Physical: physical, + Logical: logical, + }), false, true, 0) + re.NoError(err) + + // Trigger checkTSOSplit to ensure the split is finished. + testutil.Eventually(re, func() bool { + _, _, err = cli.GetTS(ctx) + re.NoError(err) + kg, err := kgm.GetKeyspaceGroupByID(mcsutils.DefaultKeyspaceGroupID) + re.NoError(err) + return !kg.IsSplitting() + }) + + // Then split keyspace group 0 to 2 with keyspace 1. + testutil.Eventually(re, func() bool { + err = kgm.SplitKeyspaceGroupByID(0, 2, []uint32{1}) + return err == nil + }) + + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) +} From 8ddf8083255b94d228d7b54bba9a04c789f2d500 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 16 Jun 2023 00:42:25 +0800 Subject: [PATCH 3/8] address comments Signed-off-by: lhy1024 --- client/client.go | 2 ++ .../mcs/tso/keyspace_group_manager_test.go | 17 +++++++---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/client/client.go b/client/client.go index eb61d0a3631..0597f7344ee 100644 --- a/client/client.go +++ b/client/client.go @@ -515,6 +515,8 @@ func newClientWithKeyspaceName( if err := c.initRetry(c.loadKeyspaceMeta, keyspaceName); err != nil { return nil, err } + // We call "c.pdSvcDiscovery.SetKeyspaceID(c.keyspaceID)" after service mode already switching to API mode + // and tso service discovery already initialized, so here we need to set the tso_service_discovery's keyspace id too. c.pdSvcDiscovery.SetKeyspaceID(c.keyspaceID) c.serviceModeKeeper.SetKeyspaceID(c.keyspaceID) log.Info("[pd] create pd client with endpoints and keyspace", diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index a91e18f1935..c7e1961a66c 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -33,7 +33,6 @@ import ( "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/storage/endpoint" tsopkg "github.com/tikv/pd/pkg/tso" - "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/server/apiv2/handlers" @@ -496,17 +495,15 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { err = tc.RunInitialServers() re.NoError(err) defer tc.Destroy() - tsoServer, tsoServerCleanup1, err := tests.StartSingleTSOTestServer(ctx, re, pdAddr, tempurl.Alloc()) - defer tsoServerCleanup1() - re.NoError(err) - _, tsoServerCleanup2, err := tests.StartSingleTSOTestServer(ctx, re, pdAddr, tempurl.Alloc()) - defer tsoServerCleanup2() - re.NoError(err) - time.Sleep(2 * time.Second) tc.WaitLeader() leaderServer := tc.GetServer(tc.GetLeader()) re.NoError(leaderServer.BootstrapCluster()) + tsoCluster, err := mcs.NewTestTSOCluster(ctx, 2, pdAddr) + re.NoError(err) + defer tsoCluster.Destroy() + tsoCluster.WaitForDefaultPrimaryServing(re) + // First split keyspace group 0 to 1 with keyspace 2. kgm := leaderServer.GetServer().GetKeyspaceGroupManager() re.NotNil(kgm) @@ -519,8 +516,8 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { cli := <-done defer cli.Close() physical, logical, err := cli.GetTS(ctx) - physical += time.Hour.Milliseconds() - tsoServer.GetHandler().ResetTS(tsoutil.GenerateTS(&pdpb.Timestamp{ + physical += time.Second.Milliseconds() + tsoCluster.GetPrimaryServer(0, 0).GetHandler().ResetTS(tsoutil.GenerateTS(&pdpb.Timestamp{ Physical: physical, Logical: logical, }), false, true, 0) From 7e4ab5a6f52dae2527f5c438e71c1d102aba5bdb Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 16 Jun 2023 01:53:19 +0800 Subject: [PATCH 4/8] fix data race Signed-off-by: lhy1024 --- pkg/balancer/round_robin.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/balancer/round_robin.go b/pkg/balancer/round_robin.go index cef35c43a5f..5013a447d3e 100644 --- a/pkg/balancer/round_robin.go +++ b/pkg/balancer/round_robin.go @@ -51,7 +51,8 @@ func (r *RoundRobin[T]) Next() (t T) { func (r *RoundRobin[T]) GetAll() []T { r.RLock() defer r.RUnlock() - return r.nodes + // return a copy to avoid data race + return append(r.nodes[:0:0], r.nodes...) } // Put puts one into balancer. From 086b0ed12c6e22cdec6c76fb893b02dec1daf7fc Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 16 Jun 2023 11:23:48 +0800 Subject: [PATCH 5/8] address comments Signed-off-by: lhy1024 --- .../mcs/tso/keyspace_group_manager_test.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index c7e1961a66c..fd1ce9a67ff 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -512,18 +512,9 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { return err == nil }) - // Set the TSO of the keyspace group 0 to a large value to make test stable. + // Trigger checkTSOSplit to ensure the split is finished. cli := <-done defer cli.Close() - physical, logical, err := cli.GetTS(ctx) - physical += time.Second.Milliseconds() - tsoCluster.GetPrimaryServer(0, 0).GetHandler().ResetTS(tsoutil.GenerateTS(&pdpb.Timestamp{ - Physical: physical, - Logical: logical, - }), false, true, 0) - re.NoError(err) - - // Trigger checkTSOSplit to ensure the split is finished. testutil.Eventually(re, func() bool { _, _, err = cli.GetTS(ctx) re.NoError(err) From 25c930a1e43ebe5d350a08d5be12d5dfd8039617 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 16 Jun 2023 11:54:26 +0800 Subject: [PATCH 6/8] add more check Signed-off-by: lhy1024 --- .../mcs/tso/keyspace_group_manager_test.go | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index fd1ce9a67ff..6e0f05d00a4 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -483,12 +483,18 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { pdAddr := tc.GetConfig().GetClientURL() // Start pd client and wait pd server start. - done := make(chan pd.Client) + var clients sync.Map go func() { apiCtx := pd.NewAPIContextV2("keyspace_b") // its keyspace id is 2. cli, err := pd.NewClientWithAPIContext(ctx, apiCtx, []string{pdAddr}, pd.SecurityOption{}) re.NoError(err) - done <- cli + clients.Store("keyspace_b", cli) + }() + go func() { + apiCtx := pd.NewAPIContextV2("keyspace_a") // its keyspace id is 1. + cli, err := pd.NewClientWithAPIContext(ctx, apiCtx, []string{pdAddr}, pd.SecurityOption{}) + re.NoError(err) + clients.Store("keyspace_a", cli) }() // Start api server and tso server. @@ -504,6 +510,20 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { defer tsoCluster.Destroy() tsoCluster.WaitForDefaultPrimaryServing(re) + // wait client ready. + testutil.Eventually(re, func() bool { + count := 0 + clients.Range(func(key, value interface{}) bool { + count++ + return true + }) + return count == 2 + }) + client_a, ok := clients.Load("keyspace_a") + re.True(ok) + client_b, ok := clients.Load("keyspace_b") + re.True(ok) + // First split keyspace group 0 to 1 with keyspace 2. kgm := leaderServer.GetServer().GetKeyspaceGroupManager() re.NotNil(kgm) @@ -513,15 +533,14 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { }) // Trigger checkTSOSplit to ensure the split is finished. - cli := <-done - defer cli.Close() testutil.Eventually(re, func() bool { - _, _, err = cli.GetTS(ctx) + _, _, err = client_b.(pd.Client).GetTS(ctx) re.NoError(err) kg, err := kgm.GetKeyspaceGroupByID(mcsutils.DefaultKeyspaceGroupID) re.NoError(err) return !kg.IsSplitting() }) + client_b.(pd.Client).Close() // Then split keyspace group 0 to 2 with keyspace 1. testutil.Eventually(re, func() bool { @@ -529,5 +548,15 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { return err == nil }) + // Trigger checkTSOSplit to ensure the split is finished. + testutil.Eventually(re, func() bool { + _, _, err = client_a.(pd.Client).GetTS(ctx) + re.NoError(err) + kg, err := kgm.GetKeyspaceGroupByID(mcsutils.DefaultKeyspaceGroupID) + re.NoError(err) + return !kg.IsSplitting() + }) + client_a.(pd.Client).Close() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) } From dbe3a6f17691eed3a1fa0c72a5e13317389386f1 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 16 Jun 2023 11:59:00 +0800 Subject: [PATCH 7/8] add more check Signed-off-by: lhy1024 --- .../mcs/tso/keyspace_group_manager_test.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index 6e0f05d00a4..01a298d5ff6 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -536,8 +536,7 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { testutil.Eventually(re, func() bool { _, _, err = client_b.(pd.Client).GetTS(ctx) re.NoError(err) - kg, err := kgm.GetKeyspaceGroupByID(mcsutils.DefaultKeyspaceGroupID) - re.NoError(err) + kg := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 0) return !kg.IsSplitting() }) client_b.(pd.Client).Close() @@ -552,11 +551,21 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { testutil.Eventually(re, func() bool { _, _, err = client_a.(pd.Client).GetTS(ctx) re.NoError(err) - kg, err := kgm.GetKeyspaceGroupByID(mcsutils.DefaultKeyspaceGroupID) - re.NoError(err) + kg := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 0) return !kg.IsSplitting() }) client_a.(pd.Client).Close() + // Check the keyspace group 0 is split to 1 and 2. + kg0 := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 0) + kg1 := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 1) + kg2 := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 2) + re.Equal([]uint32{0}, kg0.Keyspaces) + re.Equal([]uint32{2}, kg1.Keyspaces) + re.Equal([]uint32{1}, kg2.Keyspaces) + re.False(kg0.IsSplitting()) + re.False(kg1.IsSplitting()) + re.False(kg2.IsSplitting()) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) } From a29a977ff27ec7543cdcc03342491385e4708476 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 16 Jun 2023 12:03:19 +0800 Subject: [PATCH 8/8] fix lint Signed-off-by: lhy1024 --- .../mcs/tso/keyspace_group_manager_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index 01a298d5ff6..ed3bfe35280 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -510,7 +510,7 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { defer tsoCluster.Destroy() tsoCluster.WaitForDefaultPrimaryServing(re) - // wait client ready. + // Wait pd clients are ready. testutil.Eventually(re, func() bool { count := 0 clients.Range(func(key, value interface{}) bool { @@ -519,9 +519,9 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { }) return count == 2 }) - client_a, ok := clients.Load("keyspace_a") + clientA, ok := clients.Load("keyspace_a") re.True(ok) - client_b, ok := clients.Load("keyspace_b") + clientB, ok := clients.Load("keyspace_b") re.True(ok) // First split keyspace group 0 to 1 with keyspace 2. @@ -534,12 +534,12 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { // Trigger checkTSOSplit to ensure the split is finished. testutil.Eventually(re, func() bool { - _, _, err = client_b.(pd.Client).GetTS(ctx) + _, _, err = clientB.(pd.Client).GetTS(ctx) re.NoError(err) kg := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 0) return !kg.IsSplitting() }) - client_b.(pd.Client).Close() + clientB.(pd.Client).Close() // Then split keyspace group 0 to 2 with keyspace 1. testutil.Eventually(re, func() bool { @@ -549,12 +549,12 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { // Trigger checkTSOSplit to ensure the split is finished. testutil.Eventually(re, func() bool { - _, _, err = client_a.(pd.Client).GetTS(ctx) + _, _, err = clientA.(pd.Client).GetTS(ctx) re.NoError(err) kg := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 0) return !kg.IsSplitting() }) - client_a.(pd.Client).Close() + clientA.(pd.Client).Close() // Check the keyspace group 0 is split to 1 and 2. kg0 := handlersutil.MustLoadKeyspaceGroupByID(re, leaderServer, 0)