Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

pd_client: update client and global config #40955

Merged
merged 7 commits into from
Feb 3, 2023
Merged

pd_client: update client and global config #40955

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1d45ae8
update pd client
HuSharp Feb 2, 2023
45edb71
update bazel
HuSharp Feb 2, 2023
0495bc4
Merge branch 'master' into fix_cdc_compatible
HuSharp Feb 2, 2023
d2329a7
update client go
HuSharp Feb 2, 2023
cbdcc1b
Merge branch 'master' into fix_cdc_compatible
HuSharp Feb 2, 2023
412a8a3
Merge branch 'master' into fix_cdc_compatible
ti-chi-bot Feb 3, 2023
6b159a0
Merge branch 'master' into fix_cdc_compatible
HuSharp Feb 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
16 changes: 8 additions & 8 deletions DEPS.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3342,8 +3342,8 @@ def go_deps():
name = "com_github_pingcap_kvproto",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/kvproto",
sum = "h1:oYUK4V5PMlyIooU/+pPkKrJ3vELwcuuCNyKKlqSQa5c=",
version = "v0.0.0-20230131104319-a7c51106dfe7",
sum = "h1:ZiCJcEzmmF5xNgt8GIXekd3WQXI/22kzYQnrHi3Fc/4=",
version = "v0.0.0-20230201112839-2b853bed8125",
)
go_repository(
name = "com_github_pingcap_log",
Expand Down Expand Up @@ -4055,23 +4055,23 @@ def go_deps():
name = "com_github_tikv_client_go_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/client-go/v2",
sum = "h1:2BmijiUk1Hcv0z58DVk4ypwaNmgutzLc2YJm0SHPEWE=",
version = "v2.0.5-0.20230120021435-f89383775234",
sum = "h1:j2s6Gechj46t1GWxE0vZEPBqgp7sc7mb3v0srBmf5Tw=",
version = "v2.0.5-0.20230202101145-8fd09cd88cce",
)
go_repository(
name = "com_github_tikv_pd",
build_file_proto_mode = "disable",
importpath = "github.com/tikv/pd",
sum = "h1:ef+kODGby/rmF9fabJzqRM15NcGufkTRftROdy7jvAk=",
version = "v1.1.0-beta.0.20230201064005-6ca9a3398f15",
sum = "h1:iY/RztOIZ2nTbINUiLGsSv3SUGoEiub1GN0SKVKHJYg=",
version = "v1.1.0-beta.0.20230202094356-18df271ce57f",
)

go_repository(
name = "com_github_tikv_pd_client",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/pd/client",
sum = "h1:KK5bx0KLcpYUCnuQ06THPYT6QdAMfvwAtRQ0saVGD7k=",
version = "v0.0.0-20230119115149-5c518d079b93",
sum = "h1:xHPPej9Z8IrYdyLED4byOGtGm/7yhxyRUf4m93CpDyg=",
version = "v0.0.0-20230202094356-18df271ce57f",
)
go_repository(
name = "com_github_timakin_bodyclose",
Expand Down
5 changes: 1 addition & 4 deletions domain/globalconfigsync/globalconfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,6 @@ import (
"go.uber.org/zap"
)

// GlobalConfigPath as Etcd prefix
const GlobalConfigPath = "/global/config/"

// GlobalConfigSyncer is used to sync pd global config.
type GlobalConfigSyncer struct {
pd pd.Client
Expand All @@ -44,7 +41,7 @@ func (s *GlobalConfigSyncer) StoreGlobalConfig(ctx context.Context, item pd.Glob
if s.pd == nil {
return nil
}
err := s.pd.StoreGlobalConfig(ctx, GlobalConfigPath, []pd.GlobalConfigItem{item})
err := s.pd.StoreGlobalConfig(ctx, "", []pd.GlobalConfigItem{item})
Copy link
Contributor Author

@HuSharp HuSharp Feb 2, 2023 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If empty the path will be initialized in pd, do not exposed in tidb

if err != nil {
return err
}
Expand Down
11 changes: 5 additions & 6 deletions domain/globalconfigsync/globalconfig_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package globalconfigsync_test

import (
"context"
"path"
"runtime"
"testing"
"time"
Expand Down Expand Up @@ -59,10 +58,10 @@ func TestGlobalConfigSyncer(t *testing.T) {
syncer.Notify(pd.GlobalConfigItem{Name: "a", Value: "b"})
err = syncer.StoreGlobalConfig(context.Background(), <-syncer.NotifyCh)
require.NoError(t, err)
items, revision, err := client.LoadGlobalConfig(context.Background(), globalconfigsync.GlobalConfigPath)
items, revision, err := client.LoadGlobalConfig(context.Background(), []string{"a"}, "")
require.NoError(t, err)
require.Equal(t, 1, len(items))
require.Equal(t, path.Join(globalconfigsync.GlobalConfigPath, "a"), items[0].Name)
require.Equal(t, "/global/config/a", items[0].Name)
require.Equal(t, int64(0), revision)
require.Equal(t, "b", items[0].Value)
}
Expand Down Expand Up @@ -97,15 +96,15 @@ func TestStoreGlobalConfig(t *testing.T) {
client :=
store.(kv.StorageWithPD).GetPDClient()
// enable top sql will be translated to enable_resource_metering
items, _, err := client.LoadGlobalConfig(context.Background(), globalconfigsync.GlobalConfigPath)
items, _, err := client.LoadGlobalConfig(context.Background(), []string{"enable_resource_metering", "source_id"}, "")
require.NoError(t, err)
if len(items) == 2 && items[0].Value == "" {
continue
}
require.Len(t, items, 2)
require.Equal(t, items[0].Name, path.Join(globalconfigsync.GlobalConfigPath, "enable_resource_metering"))
require.Equal(t, items[0].Name, "/global/config/enable_resource_metering")
require.Equal(t, items[0].Value, "true")
require.Equal(t, items[1].Name, path.Join(globalconfigsync.GlobalConfigPath, "source_id"))
require.Equal(t, items[1].Name, "/global/config/source_id")
require.Equal(t, items[1].Value, "2")
return
}
Expand Down
8 changes: 4 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ require (
github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32
github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3
github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059
github.com/pingcap/kvproto v0.0.0-20230131104319-a7c51106dfe7
github.com/pingcap/kvproto v0.0.0-20230201112839-2b853bed8125
github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c
github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4
github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e
Expand All @@ -92,9 +92,9 @@ require (
github.com/stretchr/testify v1.8.1
github.com/tdakkota/asciicheck v0.1.1
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
github.com/tikv/client-go/v2 v2.0.5-0.20230120021435-f89383775234
github.com/tikv/pd v1.1.0-beta.0.20230201064005-6ca9a3398f15
github.com/tikv/pd/client v0.0.0-20230119115149-5c518d079b93
github.com/tikv/client-go/v2 v2.0.5-0.20230202101145-8fd09cd88cce
github.com/tikv/pd v1.1.0-beta.0.20230202094356-18df271ce57f
github.com/tikv/pd/client v0.0.0-20230202094356-18df271ce57f
github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144
github.com/twmb/murmur3 v1.1.3
github.com/uber/jaeger-client-go v2.22.1+incompatible
Expand Down
17 changes: 8 additions & 9 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1143,9 +1143,8 @@ github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059/go.mod h1:fMRU1BA1y+r89
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/kvproto v0.0.0-20230119031034-25f1909b7934/go.mod h1:+on3Lfk/fb1lXkud3XvskJumhSIEEgN2TTbMObUlrxE=
github.com/pingcap/kvproto v0.0.0-20230131104319-a7c51106dfe7 h1:oYUK4V5PMlyIooU/+pPkKrJ3vELwcuuCNyKKlqSQa5c=
github.com/pingcap/kvproto v0.0.0-20230131104319-a7c51106dfe7/go.mod h1:+on3Lfk/fb1lXkud3XvskJumhSIEEgN2TTbMObUlrxE=
github.com/pingcap/kvproto v0.0.0-20230201112839-2b853bed8125 h1:ZiCJcEzmmF5xNgt8GIXekd3WQXI/22kzYQnrHi3Fc/4=
github.com/pingcap/kvproto v0.0.0-20230201112839-2b853bed8125/go.mod h1:+on3Lfk/fb1lXkud3XvskJumhSIEEgN2TTbMObUlrxE=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
Expand Down Expand Up @@ -1308,12 +1307,12 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJf
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU=
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
github.com/tikv/client-go/v2 v2.0.5-0.20230120021435-f89383775234 h1:2BmijiUk1Hcv0z58DVk4ypwaNmgutzLc2YJm0SHPEWE=
github.com/tikv/client-go/v2 v2.0.5-0.20230120021435-f89383775234/go.mod h1:jc7J2EbNeVvU6eXmB50wAGrTPyJwdi+0ENccMXMFSpw=
github.com/tikv/pd v1.1.0-beta.0.20230201064005-6ca9a3398f15 h1:ef+kODGby/rmF9fabJzqRM15NcGufkTRftROdy7jvAk=
github.com/tikv/pd v1.1.0-beta.0.20230201064005-6ca9a3398f15/go.mod h1:UnfZJ+ykNsVbQgo/CZ3NFyHnLJ/pQGlBD0M/WgxFHMQ=
github.com/tikv/pd/client v0.0.0-20230119115149-5c518d079b93 h1:KK5bx0KLcpYUCnuQ06THPYT6QdAMfvwAtRQ0saVGD7k=
github.com/tikv/pd/client v0.0.0-20230119115149-5c518d079b93/go.mod h1:NrbwVp9afaCmJjJEwFNtEQWfCChAW1ndnwjteHHS+d0=
github.com/tikv/client-go/v2 v2.0.5-0.20230202101145-8fd09cd88cce h1:j2s6Gechj46t1GWxE0vZEPBqgp7sc7mb3v0srBmf5Tw=
github.com/tikv/client-go/v2 v2.0.5-0.20230202101145-8fd09cd88cce/go.mod h1:DtwnMX8PDLcbXn2T4AyiCFPjmzTr1F4MQzJQpQhJeLM=
github.com/tikv/pd v1.1.0-beta.0.20230202094356-18df271ce57f h1:iY/RztOIZ2nTbINUiLGsSv3SUGoEiub1GN0SKVKHJYg=
github.com/tikv/pd v1.1.0-beta.0.20230202094356-18df271ce57f/go.mod h1:jb9oq6rN4U0U3FZdvqWlpi9rZzFJxiOlvZ3aj5BTpg8=
github.com/tikv/pd/client v0.0.0-20230202094356-18df271ce57f h1:xHPPej9Z8IrYdyLED4byOGtGm/7yhxyRUf4m93CpDyg=
github.com/tikv/pd/client v0.0.0-20230202094356-18df271ce57f/go.mod h1:0fdKmj9cafPLsHAeVAcgB50Uj018WGfMC1tgvzWXXQQ=
github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 h1:kl4KhGNsJIbDHS9/4U9yQo1UcPQM0kOMJHn29EoH/Ro=
github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
Expand Down
16 changes: 8 additions & 8 deletions store/mockstore/unistore/pd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ import (
"context"
"errors"
"math"
"path"
"strings"
"sync"
"sync/atomic"

Expand Down Expand Up @@ -50,19 +48,21 @@ func newPDClient(pd *us.MockPD) *pdClient {
}
}

func (c *pdClient) LoadGlobalConfig(ctx context.Context, configPath string) ([]pd.GlobalConfigItem, int64, error) {
ret := make([]pd.GlobalConfigItem, 0)
for k, v := range c.globalConfig {
if strings.HasPrefix(k, configPath) {
ret = append(ret, pd.GlobalConfigItem{Name: k, Value: v})
func (c *pdClient) LoadGlobalConfig(ctx context.Context, names []string, configPath string) ([]pd.GlobalConfigItem, int64, error) {
ret := make([]pd.GlobalConfigItem, len(names))
for i, name := range names {
if r, ok := c.globalConfig["/global/config/"+name]; ok {
ret[i] = pd.GlobalConfigItem{Name: "/global/config/" + name, Value: r, EventType: pdpb.EventType_PUT}
} else {
ret[i] = pd.GlobalConfigItem{Name: "/global/config/" + name, Value: ""}
}
}
return ret, 0, nil
}

func (c *pdClient) StoreGlobalConfig(ctx context.Context, configPath string, items []pd.GlobalConfigItem) error {
for _, item := range items {
c.globalConfig[path.Join(configPath, item.Name)] = item.Value
c.globalConfig["/global/config/"+item.Name] = item.Value
}
return nil
}
Expand Down
36 changes: 30 additions & 6 deletions store/mockstore/unistore/pd_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,39 @@ func SetUpSuite() *GlobalConfigTestSuite {
return s
}

func TestLoadAndStore(t *testing.T) {
func TestLoad(t *testing.T) {
s := SetUpSuite()

err := s.client.StoreGlobalConfig(context.Background(), "/global/config", []pd.GlobalConfigItem{{Name: "NewObject", Value: "ok"}})
err := s.client.StoreGlobalConfig(context.Background(), "", []pd.GlobalConfigItem{{Name: "LoadOkGlobalConfig", Value: "ok"}})
require.Equal(t, nil, err)
res, _, err := s.client.LoadGlobalConfig(context.Background(), []string{"LoadOkGlobalConfig", "LoadErrGlobalConfig"}, "")
require.Equal(t, err, nil)
for _, j := range res {
println(j.Name)
switch j.Name {
case "/global/config/LoadOkGlobalConfig":
require.Equal(t, "ok", j.Value)
case "/global/config/LoadErrGlobalConfig":
require.Equal(t, "", j.Value)
default:
require.Equal(t, true, false)
}
}
s.TearDownSuite()
}

res, _, err := s.client.LoadGlobalConfig(context.Background(), "/global/config")
require.Equal(t, nil, err)
require.Equal(t, 1, len(res))
func TestStore(t *testing.T) {
s := SetUpSuite()

res, _, err := s.client.LoadGlobalConfig(context.Background(), []string{"NewObject"}, "")
require.Equal(t, err, nil)
require.Equal(t, res[0].Value, "")

err = s.client.StoreGlobalConfig(context.Background(), "", []pd.GlobalConfigItem{{Name: "NewObject", Value: "ok"}})
require.Equal(t, err, nil)

res, _, err = s.client.LoadGlobalConfig(context.Background(), []string{"NewObject"}, "")
require.Equal(t, err, nil)
require.Equal(t, res[0].Value, "ok")

s.TearDownSuite()
}
Expand Down