Skip to content

Commit

Permalink
Merge branch 'master' of github.com:tikv/pd into RawKV_GC_API_storageTTL
Browse files Browse the repository at this point in the history
  • Loading branch information
AmoebaProtozoa committed Jun 21, 2022
2 parents ffe8a18 + b100049 commit ab98f7c
Show file tree
Hide file tree
Showing 156 changed files with 6,872 additions and 6,442 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ package.list
report.xml
coverage.xml
coverage
*.txt
2 changes: 1 addition & 1 deletion .golangci.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
run:
deadline: 120s
timeout: 3m
linters:
enable:
- misspell
Expand Down
8 changes: 3 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -112,10 +112,8 @@ docker-image:
#### Build utils ###

swagger-spec: install-tools
go mod vendor
swag init --parseVendor --generalInfo server/api/router.go --exclude vendor/github.com/pingcap/tidb-dashboard --output docs/swagger
go mod tidy
rm -rf vendor
swag init --parseDependency --parseInternal --parseDepth 1 --dir server --generalInfo api/router.go --output docs/swagger
swag fmt --dir server

dashboard-ui:
./scripts/embed-dashboard-ui.sh
Expand Down Expand Up @@ -150,7 +148,7 @@ static: install-tools
@ echo "gofmt ..."
@ gofmt -s -l -d $(PACKAGE_DIRECTORIES) 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }'
@ echo "golangci-lint ..."
@ golangci-lint run $(PACKAGE_DIRECTORIES)
@ golangci-lint run --verbose $(PACKAGE_DIRECTORIES)
@ echo "revive ..."
@ revive -formatter friendly -config revive.toml $(PACKAGES)

Expand Down
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,15 @@ require (
github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a
github.com/pingcap/log v0.0.0-20210906054005-afc726e70354
github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d
github.com/pingcap/tidb-dashboard v0.0.0-20220518164040-4d621864a9a0
github.com/pingcap/tidb-dashboard v0.0.0-20220613053259-1b8920062bd3
github.com/prometheus/client_golang v1.1.0
github.com/prometheus/common v0.6.0
github.com/sasha-s/go-deadlock v0.2.0
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.7.0
github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba
github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476
github.com/swaggo/swag v1.8.3
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
github.com/tidwall/gjson v1.9.3 // indirect
github.com/unrolled/render v1.0.1
Expand All @@ -50,9 +50,9 @@ require (
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
go.uber.org/goleak v1.1.12
go.uber.org/zap v1.19.1
golang.org/x/text v0.3.3
golang.org/x/text v0.3.7
golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
golang.org/x/tools v0.1.5
golang.org/x/tools v0.1.10
google.golang.org/grpc v1.26.0
gotest.tools/gotestsum v1.7.0
)
67 changes: 48 additions & 19 deletions go.sum

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions pkg/encryption/crypter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ func TestKeyLength(t *testing.T) {
t.Parallel()
re := require.New(t)
_, err := KeyLength(encryptionpb.EncryptionMethod_PLAINTEXT)
re.NotNil(err)
re.Error(err)
_, err = KeyLength(encryptionpb.EncryptionMethod_UNKNOWN)
re.NotNil(err)
re.Error(err)
length, err := KeyLength(encryptionpb.EncryptionMethod_AES128_CTR)
re.NoError(err)
re.Equal(16, length)
Expand Down Expand Up @@ -111,5 +111,5 @@ func TestAesGcmCrypter(t *testing.T) {
// ignore overflow
fakeCiphertext[0] = ciphertext[0] + 1
_, err = AesGcmDecrypt(key, fakeCiphertext, iv)
re.NotNil(err)
re.Error(err)
}
18 changes: 7 additions & 11 deletions pkg/encryption/master_key_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,7 @@ func TestNewFileMasterKeyMissingPath(t *testing.T) {
func TestNewFileMasterKeyMissingFile(t *testing.T) {
t.Parallel()
re := require.New(t)
dir, err := os.MkdirTemp("", "test_key_files")
re.NoError(err)
dir := t.TempDir()
path := dir + "/key"
config := &encryptionpb.MasterKey{
Backend: &encryptionpb.MasterKey_File{
Expand All @@ -109,15 +108,14 @@ func TestNewFileMasterKeyMissingFile(t *testing.T) {
},
},
}
_, err = NewMasterKey(config, nil)
_, err := NewMasterKey(config, nil)
re.Error(err)
}

func TestNewFileMasterKeyNotHexString(t *testing.T) {
t.Parallel()
re := require.New(t)
dir, err := os.MkdirTemp("", "test_key_files")
re.NoError(err)
dir := t.TempDir()
path := dir + "/key"
os.WriteFile(path, []byte("not-a-hex-string"), 0600)
config := &encryptionpb.MasterKey{
Expand All @@ -127,15 +125,14 @@ func TestNewFileMasterKeyNotHexString(t *testing.T) {
},
},
}
_, err = NewMasterKey(config, nil)
_, err := NewMasterKey(config, nil)
re.Error(err)
}

func TestNewFileMasterKeyLengthMismatch(t *testing.T) {
t.Parallel()
re := require.New(t)
dir, err := os.MkdirTemp("", "test_key_files")
re.NoError(err)
dir := t.TempDir()
path := dir + "/key"
os.WriteFile(path, []byte("2f07ec61e5a50284f47f2b402a962ec6"), 0600)
config := &encryptionpb.MasterKey{
Expand All @@ -145,16 +142,15 @@ func TestNewFileMasterKeyLengthMismatch(t *testing.T) {
},
},
}
_, err = NewMasterKey(config, nil)
_, err := NewMasterKey(config, nil)
re.Error(err)
}

func TestNewFileMasterKey(t *testing.T) {
t.Parallel()
re := require.New(t)
key := "2f07ec61e5a50284f47f2b402a962ec672e500b26cb3aa568bb1531300c74806"
dir, err := os.MkdirTemp("", "test_key_files")
re.NoError(err)
dir := t.TempDir()
path := dir + "/key"
os.WriteFile(path, []byte(key), 0600)
config := &encryptionpb.MasterKey{
Expand Down
12 changes: 3 additions & 9 deletions pkg/etcdutil/etcdutil.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (
"fmt"
"net/http"
"net/url"
"os"
"testing"
"time"

"github.com/gogo/protobuf/proto"
Expand Down Expand Up @@ -182,10 +182,10 @@ func EtcdKVPutWithTTL(ctx context.Context, c *clientv3.Client, key string, value
}

// NewTestSingleConfig is used to create a etcd config for the unit test purpose.
func NewTestSingleConfig() *embed.Config {
func NewTestSingleConfig(t *testing.T) *embed.Config {
cfg := embed.NewConfig()
cfg.Name = "test_etcd"
cfg.Dir, _ = os.MkdirTemp("/tmp", "test_etcd")
cfg.Dir = t.TempDir()
cfg.WalDir = ""
cfg.Logger = "zap"
cfg.LogOutputs = []string{"stdout"}
Expand All @@ -202,9 +202,3 @@ func NewTestSingleConfig() *embed.Config {
cfg.ClusterState = embed.ClusterStateFlagNew
return cfg
}

// CleanConfig is used to clean the etcd data for the unit test purpose.
func CleanConfig(cfg *embed.Config) {
// Clean data directory
os.RemoveAll(cfg.Dir)
}
12 changes: 4 additions & 8 deletions pkg/etcdutil/etcdutil_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,10 @@ import (
func TestMemberHelpers(t *testing.T) {
t.Parallel()
re := require.New(t)
cfg1 := NewTestSingleConfig()
cfg1 := NewTestSingleConfig(t)
etcd1, err := embed.StartEtcd(cfg1)
defer func() {
etcd1.Close()
CleanConfig(cfg1)
}()
re.NoError(err)

Expand All @@ -55,7 +54,7 @@ func TestMemberHelpers(t *testing.T) {

// Test AddEtcdMember
// Make a new etcd config.
cfg2 := NewTestSingleConfig()
cfg2 := NewTestSingleConfig(t)
cfg2.Name = "etcd2"
cfg2.InitialCluster = cfg1.InitialCluster + fmt.Sprintf(",%s=%s", cfg2.Name, &cfg2.LPUrls[0])
cfg2.ClusterState = embed.ClusterStateFlagExisting
Expand All @@ -68,7 +67,6 @@ func TestMemberHelpers(t *testing.T) {
etcd2, err := embed.StartEtcd(cfg2)
defer func() {
etcd2.Close()
CleanConfig(cfg2)
}()
re.NoError(err)
re.Equal(uint64(etcd2.Server.ID()), addResp.Member.ID)
Expand Down Expand Up @@ -113,11 +111,10 @@ func TestMemberHelpers(t *testing.T) {
func TestEtcdKVGet(t *testing.T) {
t.Parallel()
re := require.New(t)
cfg := NewTestSingleConfig()
cfg := NewTestSingleConfig(t)
etcd, err := embed.StartEtcd(cfg)
defer func() {
etcd.Close()
CleanConfig(cfg)
}()
re.NoError(err)

Expand Down Expand Up @@ -165,11 +162,10 @@ func TestEtcdKVGet(t *testing.T) {
func TestEtcdKVPutWithTTL(t *testing.T) {
t.Parallel()
re := require.New(t)
cfg := NewTestSingleConfig()
cfg := NewTestSingleConfig(t)
etcd, err := embed.StartEtcd(cfg)
defer func() {
etcd.Close()
CleanConfig(cfg)
}()
re.NoError(err)

Expand Down
49 changes: 49 additions & 0 deletions pkg/jsonutil/jsonutil.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// Copyright 2022 TiKV Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jsonutil

import (
"bytes"
"encoding/json"

"github.com/tikv/pd/pkg/reflectutil"
)

// AddKeyValue is used to add a key value pair into `old`
func AddKeyValue(old interface{}, key string, value interface{}) (updated bool, found bool, err error) {
data, err := json.Marshal(map[string]interface{}{key: value})
if err != nil {
return false, false, err
}
return MergeJSONObject(old, data)
}

// MergeJSONObject is used to merge a marshaled json object into v
func MergeJSONObject(v interface{}, data []byte) (updated bool, found bool, err error) {
old, _ := json.Marshal(v)
if err := json.Unmarshal(data, v); err != nil {
return false, false, err
}
new, _ := json.Marshal(v)
if !bytes.Equal(old, new) {
return true, true, nil
}
m := make(map[string]interface{})
if err := json.Unmarshal(data, &m); err != nil {
return false, false, err
}
found = reflectutil.FindSameFieldByJSON(v, m)
return false, found, nil
}
65 changes: 65 additions & 0 deletions pkg/jsonutil/jsonutil_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// Copyright 2022 TiKV Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jsonutil

import (
"testing"

"github.com/stretchr/testify/require"
)

type testJSONStructLevel1 struct {
Name string `json:"name"`
Sub1 testJSONStructLevel2 `json:"sub1"`
Sub2 testJSONStructLevel2 `json:"sub2"`
}

type testJSONStructLevel2 struct {
SubName string `json:"sub-name"`
}

func TestJSONUtil(t *testing.T) {
t.Parallel()
re := require.New(t)
father := &testJSONStructLevel1{
Name: "father",
}
son1 := &testJSONStructLevel2{
SubName: "son1",
}
update, found, err := AddKeyValue(&father, "sub1", &son1)
re.NoError(err)
re.True(update)
re.True(found)

son2 := &testJSONStructLevel2{
SubName: "son2",
}

update, found, err = AddKeyValue(father, "sub2", &son2)
re.NoError(err)
re.True(update)
re.True(found)

update, found, err = AddKeyValue(father, "sub3", &son2)
re.NoError(err)
re.False(update)
re.False(found)

update, found, err = AddKeyValue(father, "sub2", &son2)
re.NoError(err)
re.False(update)
re.True(found)
}

0 comments on commit ab98f7c

Please sign in to comment.