From 32c44ee77f36d191a34f22361dae166d26c89f36 Mon Sep 17 00:00:00 2001 From: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 12 May 2022 16:57:55 +0800 Subject: [PATCH 01/34] added storage methods for RawKV GC Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 + server/storage/endpoint/gc_key_space.go | 164 +++++++++++++++++ server/storage/endpoint/gc_safe_point.go | 17 ++ server/storage/endpoint/key_path.go | 44 +++++ server/storage/storage_gc_test.go | 225 +++++++++++++++++++++++ tests/client/go.mod | 2 +- tests/client/go.sum | 2 + 8 files changed, 458 insertions(+), 2 deletions(-) create mode 100644 server/storage/endpoint/gc_key_space.go create mode 100644 server/storage/storage_gc_test.go diff --git a/go.mod b/go.mod index 3a45721a069..a9e7b7bfeaa 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce - github.com/pingcap/kvproto v0.0.0-20220429093005-2839fa5a1ed6 + github.com/pingcap/kvproto v0.0.0-20220512080918-81900420f7d9 github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d github.com/pingcap/tidb-dashboard v0.0.0-20220331105802-5ac69661755c diff --git a/go.sum b/go.sum index 93295382b0c..b18a05f5f55 100644 --- a/go.sum +++ b/go.sum @@ -402,6 +402,10 @@ github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20220429093005-2839fa5a1ed6 h1:gT4uxwuZzTniXdzp4mPoZjhNkDNEuZBt7HESOuLRyMI= github.com/pingcap/kvproto v0.0.0-20220429093005-2839fa5a1ed6/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E= +github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20220512080918-81900420f7d9 h1:VVmdbuGfpJVeJxtaYOUn4fp9caD8kL2EGNJDmNiB5nk= +github.com/pingcap/kvproto v0.0.0-20220512080918-81900420f7d9/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go new file mode 100644 index 00000000000..bbc9461b604 --- /dev/null +++ b/server/storage/endpoint/gc_key_space.go @@ -0,0 +1,164 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package endpoint + +import ( + "encoding/json" + "math" + "strings" + "time" + + "github.com/pingcap/errors" + "go.etcd.io/etcd/clientv3" +) + +// Predefined key spaces. More key spaces would come from "Multi-tenant". +const ( + // KeySpaceRawKVDefault is key space ID for RawKV. + KeySpaceRawKVDefault = "default_rawkv" +) + +// LoadAllKeySpaces returns a list of all key-space IDs. +// We have only predefined key-spaces by now. +// More key-spaces would come from "Multi-tenant". +func (se *StorageEndpoint) LoadAllKeySpaces() ([]*KeySpaceGCSafePoint, error) { + keySpaces := []*KeySpaceGCSafePoint{ + { + SpaceID: KeySpaceRawKVDefault, + }, + } + return keySpaces, nil +} + +// SaveServiceSafePoint saves service safe point under given key-space. +func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint) error { + if ssp.ServiceID == "" { + return errors.New("service id of service safepoint cannot be empty") + } + key := ServiceSafePointPath(spaceID, ssp.ServiceID) + value, err := json.Marshal(ssp) + if err != nil { + return err + } + return se.Save(key, string(value)) +} + +// LoadServiceSafePoint reads ServiceSafePoint for the given key-space ID and service name. +// Return nil if no safepoint not exist. +func (se *StorageEndpoint) LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) { + value, err := se.Load(ServiceSafePointPath(spaceID, serviceID)) + if err != nil || value == "" { + return nil, err + } + ssp := &ServiceSafePoint{} + if err := json.Unmarshal([]byte(value), ssp); err != nil { + return nil, err + } + return ssp, nil +} + +// LoadMinServiceSafePoint returns the minimum safepoint for the given key-space. +// Note that gc worker safe point are store separately. +// If no service safe point exist for the given key-space or all the service safe points just expired, return nil. +func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string, now time.Time) (*ServiceSafePoint, error) { + prefix := ServiceSafePointPrefix(spaceID) + prefixEnd := clientv3.GetPrefixRangeEnd(prefix) + keys, values, err := se.LoadRange(prefix, prefixEnd, 0) + if err != nil { + return nil, err + } + + min := &ServiceSafePoint{SafePoint: math.MaxUint64} + for i, key := range keys { + ssp := &ServiceSafePoint{} + if err = json.Unmarshal([]byte(values[i]), ssp); err != nil { + return nil, err + } + + // remove expired safe points. + if ssp.ExpiredAt < now.Unix() { + err = se.Remove(key) + if err != nil { + return nil, err + } + continue + } + + if ssp.SafePoint < min.SafePoint { + min = ssp + } + } + + if min.SafePoint == math.MaxUint64 { + // no service safe point or all of them are expired. + return nil, nil + } + + // successfully found a valid min safe point. + return min, nil +} + +// RemoveServiceSafePoint removes GCSafePoint for the given key-space. +func (se *StorageEndpoint) RemoveServiceSafePoint(spaceID, serviceID string) error { + key := ServiceSafePointPath(spaceID, serviceID) + return se.Remove(key) +} + +// SaveKeySpaceGCSafePoint saves GCSafePoint to the given key-space. +func (se *StorageEndpoint) SaveKeySpaceGCSafePoint(gcSafePoint *KeySpaceGCSafePoint) error { + safePoint, err := json.Marshal(gcSafePoint) + if err != nil { + return err + } + return se.Save(KeySpaceGCSafePointPath(gcSafePoint.SpaceID), string(safePoint)) +} + +// LoadKeySpaceGCSafePoint reads GCSafePoint for the given key-space. +// return nil if safepoint not exist. +func (se *StorageEndpoint) LoadKeySpaceGCSafePoint(spaceID string) (*KeySpaceGCSafePoint, error) { + value, err := se.Load(KeySpaceGCSafePointPath(spaceID)) + if err != nil || value == "" { + return nil, err + } + gcSafePoint := &KeySpaceGCSafePoint{} + if err := json.Unmarshal([]byte(value), gcSafePoint); err != nil { + return nil, err + } + return gcSafePoint, nil +} + +// LoadAllKeySpaceGCSafePoints returns slice of key-spaces and their corresponding gc safe points. +func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints() ([]*KeySpaceGCSafePoint, error) { + prefix := AllKeySpacePrefix() + prefixEnd := clientv3.GetPrefixRangeEnd(prefix) + suffix := KeySpaceGCSafePointSuffix() + keys, values, err := se.LoadRange(prefix, prefixEnd, 0) + if err != nil { + return nil, err + } + safePoints := make([]*KeySpaceGCSafePoint, 0, len(values)) + for i := range keys { + // skip non gc safe points + if !strings.HasSuffix(keys[i], suffix) { + continue + } + safePoint := &KeySpaceGCSafePoint{} + if err = json.Unmarshal([]byte(values[i]), safePoint); err != nil { + return nil, err + } + safePoints = append(safePoints, safePoint) + } + return safePoints, nil +} diff --git a/server/storage/endpoint/gc_safe_point.go b/server/storage/endpoint/gc_safe_point.go index e213eca4ed5..8d8f512d51c 100644 --- a/server/storage/endpoint/gc_safe_point.go +++ b/server/storage/endpoint/gc_safe_point.go @@ -34,6 +34,12 @@ type ServiceSafePoint struct { SafePoint uint64 `json:"safe_point"` } +// KeySpaceGCSafePoint is gcWorker's safepoint for specific key-space +type KeySpaceGCSafePoint struct { + SpaceID string `json:"space_id"` + SafePoint uint64 `json:"safe_point"` +} + // GCSafePointStorage defines the storage operations on the GC safe point. type GCSafePointStorage interface { LoadGCSafePoint() (uint64, error) @@ -42,6 +48,17 @@ type GCSafePointStorage interface { LoadAllServiceGCSafePoints() ([]*ServiceSafePoint, error) SaveServiceGCSafePoint(ssp *ServiceSafePoint) error RemoveServiceGCSafePoint(serviceID string) error + + LoadAllKeySpaces() ([]*KeySpaceGCSafePoint, error) + // Service safe point interfaces. + SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint) error + LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) + LoadMinServiceSafePoint(spaceID string, now time.Time) (*ServiceSafePoint, error) + RemoveServiceSafePoint(spaceID, serviceID string) error + // GC safe point interfaces. + SaveKeySpaceGCSafePoint(gcSafePoint *KeySpaceGCSafePoint) error + LoadKeySpaceGCSafePoint(spaceID string) (*KeySpaceGCSafePoint, error) + LoadAllKeySpaceGCSafePoints() ([]*KeySpaceGCSafePoint, error) } var _ GCSafePointStorage = (*StorageEndpoint)(nil) diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index 1f5e05601cf..be3acfcca40 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -31,6 +31,7 @@ const ( customScheduleConfigPath = "scheduler_config" gcWorkerServiceSafePointID = "gc_worker" minResolvedTS = "min_resolved_ts" + keySpacePath = "keySpace" ) // AppendToRootPath appends the given key to the rootPath. @@ -103,3 +104,46 @@ func gcSafePointServicePath(serviceID string) string { func MinResolvedTSPath() string { return path.Join(clusterPath, minResolvedTS) } + +// AllKeySpacePrefix returns prefix for all key-spaces +// Path: /key-space/ +func AllKeySpacePrefix() string { + return keySpacePath + "/" +} + +// keySpacePrefix returns path prefix for given key-space +// Prefix: /key-space/{space-id} +func keySpacePrefix(spaceID string) string { + return path.Join(keySpacePath, spaceID) +} + +// safePointPrefix returns path prefix for given key-space's safe point +// Prefix: /key-space/{space-id}/gc/safepoint +func safePointPrefix(spaceID string) string { + return path.Join(keySpacePrefix(spaceID), "gc", "safepoint") +} + +// ServiceSafePointPrefix returns the prefix of given service's service safe point +// It ends with a "/" for more precise searching +// Prefix: /key-space/{space-id}/gc/safepoint/service/ +func ServiceSafePointPrefix(spaceID string) string { + return path.Join(safePointPrefix(spaceID), "service") + "/" +} + +// KeySpaceGCSafePointPath returns the gc safe point's path of the given key-space +// Path: /key-space/{space-id}/gc/safepoint/gc +func KeySpaceGCSafePointPath(spaceID string) string { + return path.Join(safePointPrefix(spaceID), "gc") +} + +// KeySpaceGCSafePointSuffix returns the postfix for any gc safepoint path +// Postfix: /gc/safepoint/gc +func KeySpaceGCSafePointSuffix() string { + return "/" + path.Join("gc", "safepoint", "gc") +} + +// ServiceSafePointPath returns the path of given service's service safe point +// Path: /key-space/{space-id}/gc/safepoint/service/{service-id} +func ServiceSafePointPath(spaceID, serviceID string) string { + return path.Join(safePointPrefix(spaceID), "service", serviceID) +} diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go new file mode 100644 index 00000000000..8a4cc86eb81 --- /dev/null +++ b/server/storage/storage_gc_test.go @@ -0,0 +1,225 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "encoding/json" + "math" + "time" + + . "github.com/pingcap/check" + "github.com/tikv/pd/server/storage/endpoint" +) + +var _ = Suite(&testStorageGCSuite{}) + +type testStorageGCSuite struct { +} + +func testGCSafePoints() []*endpoint.KeySpaceGCSafePoint { + return []*endpoint.KeySpaceGCSafePoint{ + { + SpaceID: "KeySpace1", + SafePoint: 0, + }, + { + SpaceID: "KeySpace2", + SafePoint: 1, + }, + { + SpaceID: "KeySpace3", + SafePoint: 4396, + }, + { + SpaceID: "KeySpace4", + SafePoint: 23333333333, + }, + { + SpaceID: "testKeySpace5", + SafePoint: math.MaxUint64, + }, + } +} + +func (s *testStorageGCSuite) TestLoadKeySpaceGCSafePoint(c *C) { + storage := NewStorageWithMemoryBackend() + testData := testGCSafePoints() + r, e := storage.LoadKeySpaceGCSafePoint("testKeySpace") + c.Assert(r, IsNil) + c.Assert(e, IsNil) + for _, safePoint := range testData { + err := storage.SaveKeySpaceGCSafePoint(safePoint) + c.Assert(err, IsNil) + loaded, err := storage.LoadKeySpaceGCSafePoint(safePoint.SpaceID) + c.Assert(err, IsNil) + c.Assert(safePoint, DeepEquals, loaded) + } +} + +func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { + storage := NewStorageWithMemoryBackend() + testData := testGCSafePoints() + for _, safePoint := range testData { + err := storage.SaveKeySpaceGCSafePoint(safePoint) + c.Assert(err, IsNil) + } + gcSafePoints, err := storage.LoadAllKeySpaceGCSafePoints() + c.Assert(err, IsNil) + for i, safePoint := range testData { + c.Assert(gcSafePoints[i], DeepEquals, safePoint) + } + + // saving some service safe points. + expireAt := time.Now().Add(100 * time.Second).Unix() + serviceSafePoints := []*endpoint.ServiceSafePoint{ + {ServiceID: "1", ExpiredAt: expireAt, SafePoint: 1}, + {ServiceID: "2", ExpiredAt: expireAt, SafePoint: 2}, + {ServiceID: "3", ExpiredAt: expireAt, SafePoint: 3}, + } + spaceIDs := []string{ + "keySpace1", + "keySpace2", + "keySpace3", + } + + for _, spaceID := range spaceIDs { + for _, serviceSafePoint := range serviceSafePoints { + c.Assert(storage.SaveServiceSafePoint(spaceID, serviceSafePoint), IsNil) + } + } + + // verify that service safe points does not interfere with gc safe points. + gcSafePoints, err = storage.LoadAllKeySpaceGCSafePoints() + c.Assert(err, IsNil) + for i, safePoint := range testData { + c.Assert(gcSafePoints[i], DeepEquals, safePoint) + } + +} + +func (s *testStorageGCSuite) TestLoadAllKeySpaces(c *C) { + storage := NewStorageWithMemoryBackend() + keySpaces, err := storage.LoadAllKeySpaces() + c.Assert(err, IsNil) + c.Assert(keySpaces, DeepEquals, []*endpoint.KeySpaceGCSafePoint{{SpaceID: endpoint.KeySpaceRawKVDefault}}) +} + +func (s *testStorageGCSuite) TestLoadServiceSafePoint(c *C) { + storage := NewStorageWithMemoryBackend() + expireAt := time.Now().Add(100 * time.Second).Unix() + serviceSafePoints := []*endpoint.ServiceSafePoint{ + {ServiceID: "1", ExpiredAt: expireAt, SafePoint: 1}, + {ServiceID: "2", ExpiredAt: expireAt, SafePoint: 2}, + {ServiceID: "3", ExpiredAt: expireAt, SafePoint: 3}, + } + spaceIDs := []string{ + "keySpace1", + "keySpace2", + "keySpace3", + } + + for _, spaceID := range spaceIDs { + for _, serviceSafePoint := range serviceSafePoints { + c.Assert(storage.SaveServiceSafePoint(spaceID, serviceSafePoint), IsNil) + } + } + for _, spaceID := range spaceIDs { + for _, serviceSafePoint := range serviceSafePoints { + key := endpoint.ServiceSafePointPath(spaceID, serviceSafePoint.ServiceID) + value, err := storage.Load(key) + c.Assert(err, IsNil) + ssp := &endpoint.ServiceSafePoint{} + c.Assert(json.Unmarshal([]byte(value), ssp), IsNil) + c.Assert(ssp, DeepEquals, serviceSafePoint) + } + } +} + +func (s *testStorageGCSuite) TestRemoveServiceSafePoint(c *C) { + storage := NewStorageWithMemoryBackend() + expireAt := time.Now().Add(100 * time.Second).Unix() + + serviceSafePoints := []*endpoint.ServiceSafePoint{ + {ServiceID: "1", ExpiredAt: expireAt, SafePoint: 1}, + {ServiceID: "2", ExpiredAt: expireAt, SafePoint: 2}, + {ServiceID: "3", ExpiredAt: expireAt, SafePoint: 3}, + } + spaceIDs := []string{ + "keySpace1", + "keySpace2", + "keySpace3", + } + // save service safe points + for _, spaceID := range spaceIDs { + for _, serviceSafePoint := range serviceSafePoints { + c.Assert(storage.SaveServiceSafePoint(spaceID, serviceSafePoint), IsNil) + } + } + + // remove service safe points + for _, spaceID := range spaceIDs { + for _, serviceSafePoint := range serviceSafePoints { + c.Assert(storage.RemoveServiceSafePoint(spaceID, serviceSafePoint.ServiceID), IsNil) + } + } + + // check that service safe points are empty + for _, spaceID := range spaceIDs { + for _, serviceSafePoint := range serviceSafePoints { + safepoint, err := storage.LoadServiceSafePoint(spaceID, serviceSafePoint.ServiceID) + c.Assert(err, IsNil) + c.Assert(safepoint, IsNil) + } + } +} + +func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { + storage := NewStorageWithMemoryBackend() + currentTime := time.Now() + expireAt1 := currentTime.Add(100 * time.Second).Unix() + expireAt2 := currentTime.Add(200 * time.Second).Unix() + expireAt3 := currentTime.Add(300 * time.Second).Unix() + + serviceSafePoints := []*endpoint.ServiceSafePoint{ + {ServiceID: "0", ExpiredAt: expireAt1, SafePoint: 100}, + {ServiceID: "1", ExpiredAt: expireAt2, SafePoint: 200}, + {ServiceID: "2", ExpiredAt: expireAt3, SafePoint: 300}, + } + + testKeySpace := "test" + for _, serviceSafePoint := range serviceSafePoints { + c.Assert(storage.SaveServiceSafePoint(testKeySpace, serviceSafePoint), IsNil) + } + minSafePoint, err := storage.LoadMinServiceSafePoint(testKeySpace, currentTime) + c.Assert(err, IsNil) + c.Assert(minSafePoint, DeepEquals, serviceSafePoints[0]) + + // this should remove safePoint with ServiceID 0 due to expiration + // and find the safePoint with ServiceID 1 + minSafePoint2, err := storage.LoadMinServiceSafePoint(testKeySpace, currentTime.Add(150*time.Second)) + c.Assert(err, IsNil) + c.Assert(minSafePoint2, DeepEquals, serviceSafePoints[1]) + + // verify that service safe point with ServiceID 0 has been removed + ssp, err := storage.LoadServiceSafePoint(testKeySpace, "0") + c.Assert(err, IsNil) + c.Assert(ssp, IsNil) + + // this should remove all service safe points + // and return nil + ssp, err = storage.LoadMinServiceSafePoint(testKeySpace, currentTime.Add(500*time.Second)) + c.Assert(err, IsNil) + c.Assert(ssp, IsNil) +} diff --git a/tests/client/go.mod b/tests/client/go.mod index ddd1ce30edb..c5ff9d17bbb 100644 --- a/tests/client/go.mod +++ b/tests/client/go.mod @@ -7,7 +7,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20220429093005-2839fa5a1ed6 + github.com/pingcap/kvproto v0.0.0-20220512080918-81900420f7d9 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 diff --git a/tests/client/go.sum b/tests/client/go.sum index 8441c52a709..39cfdd3ddc8 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -411,6 +411,8 @@ github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLy github.com/pingcap/kvproto v0.0.0-20220330070404-8c4cd3f93748/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/kvproto v0.0.0-20220429093005-2839fa5a1ed6 h1:gT4uxwuZzTniXdzp4mPoZjhNkDNEuZBt7HESOuLRyMI= github.com/pingcap/kvproto v0.0.0-20220429093005-2839fa5a1ed6/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20220512080918-81900420f7d9 h1:VVmdbuGfpJVeJxtaYOUn4fp9caD8kL2EGNJDmNiB5nk= +github.com/pingcap/kvproto v0.0.0-20220512080918-81900420f7d9/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= From b12eb9bc04a2eae59c059cee561dc6afa2726dd2 Mon Sep 17 00:00:00 2001 From: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 12 May 2022 19:38:14 +0800 Subject: [PATCH 02/34] push back on updating kvproto in go.mod Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ---- tests/client/go.mod | 2 +- tests/client/go.sum | 2 -- 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index a9e7b7bfeaa..3a45721a069 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce - github.com/pingcap/kvproto v0.0.0-20220512080918-81900420f7d9 + github.com/pingcap/kvproto v0.0.0-20220429093005-2839fa5a1ed6 github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d github.com/pingcap/tidb-dashboard v0.0.0-20220331105802-5ac69661755c diff --git a/go.sum b/go.sum index b18a05f5f55..93295382b0c 100644 --- a/go.sum +++ b/go.sum @@ -402,10 +402,6 @@ github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20220429093005-2839fa5a1ed6 h1:gT4uxwuZzTniXdzp4mPoZjhNkDNEuZBt7HESOuLRyMI= github.com/pingcap/kvproto v0.0.0-20220429093005-2839fa5a1ed6/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20220512080918-81900420f7d9 h1:VVmdbuGfpJVeJxtaYOUn4fp9caD8kL2EGNJDmNiB5nk= -github.com/pingcap/kvproto v0.0.0-20220512080918-81900420f7d9/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= diff --git a/tests/client/go.mod b/tests/client/go.mod index c5ff9d17bbb..ddd1ce30edb 100644 --- a/tests/client/go.mod +++ b/tests/client/go.mod @@ -7,7 +7,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20220512080918-81900420f7d9 + github.com/pingcap/kvproto v0.0.0-20220429093005-2839fa5a1ed6 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 diff --git a/tests/client/go.sum b/tests/client/go.sum index 39cfdd3ddc8..8441c52a709 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -411,8 +411,6 @@ github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLy github.com/pingcap/kvproto v0.0.0-20220330070404-8c4cd3f93748/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/kvproto v0.0.0-20220429093005-2839fa5a1ed6 h1:gT4uxwuZzTniXdzp4mPoZjhNkDNEuZBt7HESOuLRyMI= github.com/pingcap/kvproto v0.0.0-20220429093005-2839fa5a1ed6/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20220512080918-81900420f7d9 h1:VVmdbuGfpJVeJxtaYOUn4fp9caD8kL2EGNJDmNiB5nk= -github.com/pingcap/kvproto v0.0.0-20220512080918-81900420f7d9/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= From d0b4f3bc82a9e83f5ddb4639c5ef4745134c2cdb Mon Sep 17 00:00:00 2001 From: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 12 May 2022 19:45:43 +0800 Subject: [PATCH 03/34] linting Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/storage_gc_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index 8a4cc86eb81..b057489631e 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -106,7 +106,6 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { for i, safePoint := range testData { c.Assert(gcSafePoints[i], DeepEquals, safePoint) } - } func (s *testStorageGCSuite) TestLoadAllKeySpaces(c *C) { From a2ba0e763872fd621ea6ecfe8a910e154df8c230 Mon Sep 17 00:00:00 2001 From: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 12 May 2022 20:22:36 +0800 Subject: [PATCH 04/34] changed storage path structure Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 4 +- server/storage/endpoint/key_path.go | 51 ++++++++++--------------- server/storage/storage_gc_test.go | 2 +- 3 files changed, 23 insertions(+), 34 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index bbc9461b604..b54a273d8fa 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -141,9 +141,9 @@ func (se *StorageEndpoint) LoadKeySpaceGCSafePoint(spaceID string) (*KeySpaceGCS // LoadAllKeySpaceGCSafePoints returns slice of key-spaces and their corresponding gc safe points. func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints() ([]*KeySpaceGCSafePoint, error) { - prefix := AllKeySpacePrefix() + prefix := SafePointPrefix() prefixEnd := clientv3.GetPrefixRangeEnd(prefix) - suffix := KeySpaceGCSafePointSuffix() + suffix := GCSafePointSuffix() keys, values, err := se.LoadRange(prefix, prefixEnd, 0) if err != nil { return nil, err diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index be3acfcca40..4f924e6ba6b 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -31,7 +31,8 @@ const ( customScheduleConfigPath = "scheduler_config" gcWorkerServiceSafePointID = "gc_worker" minResolvedTS = "min_resolved_ts" - keySpacePath = "keySpace" + keySpaceGCPrefix = "key_space/gc_safepoint" + gcSafePointSuffix = "gc" ) // AppendToRootPath appends the given key to the rootPath. @@ -105,45 +106,33 @@ func MinResolvedTSPath() string { return path.Join(clusterPath, minResolvedTS) } -// AllKeySpacePrefix returns prefix for all key-spaces -// Path: /key-space/ -func AllKeySpacePrefix() string { - return keySpacePath + "/" -} - -// keySpacePrefix returns path prefix for given key-space -// Prefix: /key-space/{space-id} -func keySpacePrefix(spaceID string) string { - return path.Join(keySpacePath, spaceID) -} - -// safePointPrefix returns path prefix for given key-space's safe point -// Prefix: /key-space/{space-id}/gc/safepoint -func safePointPrefix(spaceID string) string { - return path.Join(keySpacePrefix(spaceID), "gc", "safepoint") -} - // ServiceSafePointPrefix returns the prefix of given service's service safe point // It ends with a "/" for more precise searching -// Prefix: /key-space/{space-id}/gc/safepoint/service/ +// Prefix: /key-space/gc-safepoint/{space-id}/service/ func ServiceSafePointPrefix(spaceID string) string { - return path.Join(safePointPrefix(spaceID), "service") + "/" + return path.Join(keySpaceGCPrefix, spaceID, "service") + "/" } // KeySpaceGCSafePointPath returns the gc safe point's path of the given key-space -// Path: /key-space/{space-id}/gc/safepoint/gc +// Path: /key_space/gc_safepoint/{space_id}/gc func KeySpaceGCSafePointPath(spaceID string) string { - return path.Join(safePointPrefix(spaceID), "gc") -} - -// KeySpaceGCSafePointSuffix returns the postfix for any gc safepoint path -// Postfix: /gc/safepoint/gc -func KeySpaceGCSafePointSuffix() string { - return "/" + path.Join("gc", "safepoint", "gc") + return path.Join(keySpaceGCPrefix, spaceID, gcSafePointSuffix) } // ServiceSafePointPath returns the path of given service's service safe point -// Path: /key-space/{space-id}/gc/safepoint/service/{service-id} +// Path: /key_space/gc_safepoint/{space_id}/service/{service-id} func ServiceSafePointPath(spaceID, serviceID string) string { - return path.Join(safePointPrefix(spaceID), "service", serviceID) + return path.Join(ServiceSafePointPrefix(spaceID), serviceID) +} + +// SafePointPrefix returns prefix for all key-spaces +// Path: /key-space/gc-safepoint/ +func SafePointPrefix() string { + return keySpaceGCPrefix + "/" +} + +// GCSafePointSuffix returns the postfix for any gc safepoint path +// Postfix: /gc +func GCSafePointSuffix() string { + return "/" + gcSafePointSuffix } diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index b057489631e..6be5dde6a26 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -47,7 +47,7 @@ func testGCSafePoints() []*endpoint.KeySpaceGCSafePoint { SafePoint: 23333333333, }, { - SpaceID: "testKeySpace5", + SpaceID: "KeySpace5", SafePoint: math.MaxUint64, }, } From b9bb3e4740f2e1dd37e226b2e95399bcebf5f850 Mon Sep 17 00:00:00 2001 From: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 12 May 2022 20:25:53 +0800 Subject: [PATCH 05/34] update comments Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/key_path.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index 4f924e6ba6b..f2cc2d25d1d 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -106,32 +106,31 @@ func MinResolvedTSPath() string { return path.Join(clusterPath, minResolvedTS) } -// ServiceSafePointPrefix returns the prefix of given service's service safe point -// It ends with a "/" for more precise searching -// Prefix: /key-space/gc-safepoint/{space-id}/service/ +// ServiceSafePointPrefix returns the prefix of given service's service safe point. +// Prefix: /key_space/gc_safepoint/{space_id}/service/ func ServiceSafePointPrefix(spaceID string) string { return path.Join(keySpaceGCPrefix, spaceID, "service") + "/" } -// KeySpaceGCSafePointPath returns the gc safe point's path of the given key-space +// KeySpaceGCSafePointPath returns the gc safe point's path of the given key-space. // Path: /key_space/gc_safepoint/{space_id}/gc func KeySpaceGCSafePointPath(spaceID string) string { return path.Join(keySpaceGCPrefix, spaceID, gcSafePointSuffix) } -// ServiceSafePointPath returns the path of given service's service safe point -// Path: /key_space/gc_safepoint/{space_id}/service/{service-id} +// ServiceSafePointPath returns the path of given service's service safe point. +// Path: /key_space/gc_safepoint/{space_id}/service/{service_id} func ServiceSafePointPath(spaceID, serviceID string) string { return path.Join(ServiceSafePointPrefix(spaceID), serviceID) } -// SafePointPrefix returns prefix for all key-spaces -// Path: /key-space/gc-safepoint/ +// SafePointPrefix returns prefix for all key-spaces' safe points. +// Path: /key_space/gc_safepoint/ func SafePointPrefix() string { return keySpaceGCPrefix + "/" } -// GCSafePointSuffix returns the postfix for any gc safepoint path +// GCSafePointSuffix returns the suffix for any gc safepoint. // Postfix: /gc func GCSafePointSuffix() string { return "/" + gcSafePointSuffix From a7e3ece800eb433ef400011836c98c636120c01a Mon Sep 17 00:00:00 2001 From: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 12 May 2022 22:20:57 +0800 Subject: [PATCH 06/34] added ByKeySpace suffix for disambiguity Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 78 +++++++++++++++--------- server/storage/endpoint/gc_safe_point.go | 17 ------ server/storage/endpoint/key_path.go | 30 ++++----- server/storage/storage.go | 1 + server/storage/storage_gc_test.go | 38 ++++++------ 5 files changed, 87 insertions(+), 77 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index b54a273d8fa..f17e2a51c9d 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -30,24 +30,33 @@ const ( KeySpaceRawKVDefault = "default_rawkv" ) -// LoadAllKeySpaces returns a list of all key-space IDs. -// We have only predefined key-spaces by now. -// More key-spaces would come from "Multi-tenant". -func (se *StorageEndpoint) LoadAllKeySpaces() ([]*KeySpaceGCSafePoint, error) { - keySpaces := []*KeySpaceGCSafePoint{ - { - SpaceID: KeySpaceRawKVDefault, - }, - } - return keySpaces, nil +// KeySpaceGCSafePoint is gcWorker's safepoint for specific key-space +type KeySpaceGCSafePoint struct { + SpaceID string `json:"space_id"` + SafePoint uint64 `json:"safe_point"` +} + +// KeySpaceGCSafePointStorage defines the storage operations on KeySpaces' safe points +type KeySpaceGCSafePointStorage interface { + // Service safe point interfaces. + SaveServiceSafePointByKeySpace(spaceID string, ssp *ServiceSafePoint) error + LoadServiceSafePointByKeySpace(spaceID, serviceID string) (*ServiceSafePoint, error) + LoadMinServiceSafePointByKeySpace(spaceID string, now time.Time) (*ServiceSafePoint, error) + RemoveServiceSafePointByKeySpace(spaceID, serviceID string) error + // GC safe point interfaces. + SaveGCSafePointByKeySpace(gcSafePoint *KeySpaceGCSafePoint) error + LoadGCSafePointByKeySpace(spaceID string) (*KeySpaceGCSafePoint, error) + LoadAllKeySpaceGCSafePoints() ([]*KeySpaceGCSafePoint, error) } -// SaveServiceSafePoint saves service safe point under given key-space. -func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint) error { +var _ KeySpaceGCSafePointStorage = (*StorageEndpoint)(nil) + +// SaveServiceSafePointByKeySpace saves service safe point under given key-space. +func (se *StorageEndpoint) SaveServiceSafePointByKeySpace(spaceID string, ssp *ServiceSafePoint) error { if ssp.ServiceID == "" { return errors.New("service id of service safepoint cannot be empty") } - key := ServiceSafePointPath(spaceID, ssp.ServiceID) + key := KeySpaceServiceSafePointPath(spaceID, ssp.ServiceID) value, err := json.Marshal(ssp) if err != nil { return err @@ -55,10 +64,10 @@ func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafe return se.Save(key, string(value)) } -// LoadServiceSafePoint reads ServiceSafePoint for the given key-space ID and service name. +// LoadServiceSafePointByKeySpace reads ServiceSafePoint for the given key-space ID and service name. // Return nil if no safepoint not exist. -func (se *StorageEndpoint) LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) { - value, err := se.Load(ServiceSafePointPath(spaceID, serviceID)) +func (se *StorageEndpoint) LoadServiceSafePointByKeySpace(spaceID, serviceID string) (*ServiceSafePoint, error) { + value, err := se.Load(KeySpaceServiceSafePointPath(spaceID, serviceID)) if err != nil || value == "" { return nil, err } @@ -69,11 +78,11 @@ func (se *StorageEndpoint) LoadServiceSafePoint(spaceID, serviceID string) (*Ser return ssp, nil } -// LoadMinServiceSafePoint returns the minimum safepoint for the given key-space. +// LoadMinServiceSafePointByKeySpace returns the minimum safepoint for the given key-space. // Note that gc worker safe point are store separately. // If no service safe point exist for the given key-space or all the service safe points just expired, return nil. -func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string, now time.Time) (*ServiceSafePoint, error) { - prefix := ServiceSafePointPrefix(spaceID) +func (se *StorageEndpoint) LoadMinServiceSafePointByKeySpace(spaceID string, now time.Time) (*ServiceSafePoint, error) { + prefix := KeySpaceServiceSafePointPrefix(spaceID) prefixEnd := clientv3.GetPrefixRangeEnd(prefix) keys, values, err := se.LoadRange(prefix, prefixEnd, 0) if err != nil { @@ -110,14 +119,14 @@ func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string, now time.Time return min, nil } -// RemoveServiceSafePoint removes GCSafePoint for the given key-space. -func (se *StorageEndpoint) RemoveServiceSafePoint(spaceID, serviceID string) error { - key := ServiceSafePointPath(spaceID, serviceID) +// RemoveServiceSafePointByKeySpace removes GCSafePoint for the given key-space. +func (se *StorageEndpoint) RemoveServiceSafePointByKeySpace(spaceID, serviceID string) error { + key := KeySpaceServiceSafePointPath(spaceID, serviceID) return se.Remove(key) } -// SaveKeySpaceGCSafePoint saves GCSafePoint to the given key-space. -func (se *StorageEndpoint) SaveKeySpaceGCSafePoint(gcSafePoint *KeySpaceGCSafePoint) error { +// SaveGCSafePointByKeySpace saves GCSafePoint to the given key-space. +func (se *StorageEndpoint) SaveGCSafePointByKeySpace(gcSafePoint *KeySpaceGCSafePoint) error { safePoint, err := json.Marshal(gcSafePoint) if err != nil { return err @@ -125,9 +134,9 @@ func (se *StorageEndpoint) SaveKeySpaceGCSafePoint(gcSafePoint *KeySpaceGCSafePo return se.Save(KeySpaceGCSafePointPath(gcSafePoint.SpaceID), string(safePoint)) } -// LoadKeySpaceGCSafePoint reads GCSafePoint for the given key-space. +// LoadGCSafePointByKeySpace reads GCSafePoint for the given key-space. // return nil if safepoint not exist. -func (se *StorageEndpoint) LoadKeySpaceGCSafePoint(spaceID string) (*KeySpaceGCSafePoint, error) { +func (se *StorageEndpoint) LoadGCSafePointByKeySpace(spaceID string) (*KeySpaceGCSafePoint, error) { value, err := se.Load(KeySpaceGCSafePointPath(spaceID)) if err != nil || value == "" { return nil, err @@ -140,10 +149,11 @@ func (se *StorageEndpoint) LoadKeySpaceGCSafePoint(spaceID string) (*KeySpaceGCS } // LoadAllKeySpaceGCSafePoints returns slice of key-spaces and their corresponding gc safe points. +// It also returns spaceID of any default key spaces that do not have a gc safepoint. func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints() ([]*KeySpaceGCSafePoint, error) { - prefix := SafePointPrefix() + prefix := KeySpaceSafePointPrefix() prefixEnd := clientv3.GetPrefixRangeEnd(prefix) - suffix := GCSafePointSuffix() + suffix := KeySpaceGCSafePointSuffix() keys, values, err := se.LoadRange(prefix, prefixEnd, 0) if err != nil { return nil, err @@ -160,5 +170,17 @@ func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints() ([]*KeySpaceGCSafePoint } safePoints = append(safePoints, safePoint) } + + // make sure all default key spaces are included in result + defaultKeySpaces := []string{KeySpaceRawKVDefault} + for _, defaultKeySpace := range defaultKeySpaces { + value, err := se.Load(KeySpaceGCSafePointPath(defaultKeySpace)) + if err != nil { + return nil, err + } + if value == "" { + safePoints = append(safePoints, &KeySpaceGCSafePoint{SpaceID: defaultKeySpace}) + } + } return safePoints, nil } diff --git a/server/storage/endpoint/gc_safe_point.go b/server/storage/endpoint/gc_safe_point.go index 8d8f512d51c..e213eca4ed5 100644 --- a/server/storage/endpoint/gc_safe_point.go +++ b/server/storage/endpoint/gc_safe_point.go @@ -34,12 +34,6 @@ type ServiceSafePoint struct { SafePoint uint64 `json:"safe_point"` } -// KeySpaceGCSafePoint is gcWorker's safepoint for specific key-space -type KeySpaceGCSafePoint struct { - SpaceID string `json:"space_id"` - SafePoint uint64 `json:"safe_point"` -} - // GCSafePointStorage defines the storage operations on the GC safe point. type GCSafePointStorage interface { LoadGCSafePoint() (uint64, error) @@ -48,17 +42,6 @@ type GCSafePointStorage interface { LoadAllServiceGCSafePoints() ([]*ServiceSafePoint, error) SaveServiceGCSafePoint(ssp *ServiceSafePoint) error RemoveServiceGCSafePoint(serviceID string) error - - LoadAllKeySpaces() ([]*KeySpaceGCSafePoint, error) - // Service safe point interfaces. - SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint) error - LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) - LoadMinServiceSafePoint(spaceID string, now time.Time) (*ServiceSafePoint, error) - RemoveServiceSafePoint(spaceID, serviceID string) error - // GC safe point interfaces. - SaveKeySpaceGCSafePoint(gcSafePoint *KeySpaceGCSafePoint) error - LoadKeySpaceGCSafePoint(spaceID string) (*KeySpaceGCSafePoint, error) - LoadAllKeySpaceGCSafePoints() ([]*KeySpaceGCSafePoint, error) } var _ GCSafePointStorage = (*StorageEndpoint)(nil) diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index f2cc2d25d1d..0fd7d0c5f87 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -31,8 +31,8 @@ const ( customScheduleConfigPath = "scheduler_config" gcWorkerServiceSafePointID = "gc_worker" minResolvedTS = "min_resolved_ts" - keySpaceGCPrefix = "key_space/gc_safepoint" - gcSafePointSuffix = "gc" + keySpaceSafePointPrefix = "key_space/gc_safepoint" + keySpaceGCSafePointSuffix = "gc" ) // AppendToRootPath appends the given key to the rootPath. @@ -106,32 +106,32 @@ func MinResolvedTSPath() string { return path.Join(clusterPath, minResolvedTS) } -// ServiceSafePointPrefix returns the prefix of given service's service safe point. +// KeySpaceServiceSafePointPrefix returns the prefix of given service's service safe point. // Prefix: /key_space/gc_safepoint/{space_id}/service/ -func ServiceSafePointPrefix(spaceID string) string { - return path.Join(keySpaceGCPrefix, spaceID, "service") + "/" +func KeySpaceServiceSafePointPrefix(spaceID string) string { + return path.Join(keySpaceSafePointPrefix, spaceID, "service") + "/" } // KeySpaceGCSafePointPath returns the gc safe point's path of the given key-space. // Path: /key_space/gc_safepoint/{space_id}/gc func KeySpaceGCSafePointPath(spaceID string) string { - return path.Join(keySpaceGCPrefix, spaceID, gcSafePointSuffix) + return path.Join(keySpaceSafePointPrefix, spaceID, keySpaceGCSafePointSuffix) } -// ServiceSafePointPath returns the path of given service's service safe point. +// KeySpaceServiceSafePointPath returns the path of given service's service safe point. // Path: /key_space/gc_safepoint/{space_id}/service/{service_id} -func ServiceSafePointPath(spaceID, serviceID string) string { - return path.Join(ServiceSafePointPrefix(spaceID), serviceID) +func KeySpaceServiceSafePointPath(spaceID, serviceID string) string { + return path.Join(KeySpaceServiceSafePointPrefix(spaceID), serviceID) } -// SafePointPrefix returns prefix for all key-spaces' safe points. +// KeySpaceSafePointPrefix returns prefix for all key-spaces' safe points. // Path: /key_space/gc_safepoint/ -func SafePointPrefix() string { - return keySpaceGCPrefix + "/" +func KeySpaceSafePointPrefix() string { + return keySpaceSafePointPrefix + "/" } -// GCSafePointSuffix returns the suffix for any gc safepoint. +// KeySpaceGCSafePointSuffix returns the suffix for any gc safepoint. // Postfix: /gc -func GCSafePointSuffix() string { - return "/" + gcSafePointSuffix +func KeySpaceGCSafePointSuffix() string { + return "/" + keySpaceGCSafePointSuffix } diff --git a/server/storage/storage.go b/server/storage/storage.go index 45fc75b7ba9..31d06dd7d25 100644 --- a/server/storage/storage.go +++ b/server/storage/storage.go @@ -38,6 +38,7 @@ type Storage interface { endpoint.ReplicationStatusStorage endpoint.GCSafePointStorage endpoint.MinResolvedTSStorage + endpoint.KeySpaceGCSafePointStorage } // NewStorageWithMemoryBackend creates a new storage with memory backend. diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index 6be5dde6a26..6ec13624e0f 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -50,19 +50,23 @@ func testGCSafePoints() []*endpoint.KeySpaceGCSafePoint { SpaceID: "KeySpace5", SafePoint: math.MaxUint64, }, + { + SpaceID: endpoint.KeySpaceRawKVDefault, + SafePoint: 3000, + }, } } -func (s *testStorageGCSuite) TestLoadKeySpaceGCSafePoint(c *C) { +func (s *testStorageGCSuite) TestLoadGCSafePointByKeySpace(c *C) { storage := NewStorageWithMemoryBackend() testData := testGCSafePoints() - r, e := storage.LoadKeySpaceGCSafePoint("testKeySpace") + r, e := storage.LoadGCSafePointByKeySpace("testKeySpace") c.Assert(r, IsNil) c.Assert(e, IsNil) for _, safePoint := range testData { - err := storage.SaveKeySpaceGCSafePoint(safePoint) + err := storage.SaveGCSafePointByKeySpace(safePoint) c.Assert(err, IsNil) - loaded, err := storage.LoadKeySpaceGCSafePoint(safePoint.SpaceID) + loaded, err := storage.LoadGCSafePointByKeySpace(safePoint.SpaceID) c.Assert(err, IsNil) c.Assert(safePoint, DeepEquals, loaded) } @@ -72,7 +76,7 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { storage := NewStorageWithMemoryBackend() testData := testGCSafePoints() for _, safePoint := range testData { - err := storage.SaveKeySpaceGCSafePoint(safePoint) + err := storage.SaveGCSafePointByKeySpace(safePoint) c.Assert(err, IsNil) } gcSafePoints, err := storage.LoadAllKeySpaceGCSafePoints() @@ -96,7 +100,7 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { for _, spaceID := range spaceIDs { for _, serviceSafePoint := range serviceSafePoints { - c.Assert(storage.SaveServiceSafePoint(spaceID, serviceSafePoint), IsNil) + c.Assert(storage.SaveServiceSafePointByKeySpace(spaceID, serviceSafePoint), IsNil) } } @@ -110,7 +114,7 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { func (s *testStorageGCSuite) TestLoadAllKeySpaces(c *C) { storage := NewStorageWithMemoryBackend() - keySpaces, err := storage.LoadAllKeySpaces() + keySpaces, err := storage.LoadAllKeySpaceGCSafePoints() c.Assert(err, IsNil) c.Assert(keySpaces, DeepEquals, []*endpoint.KeySpaceGCSafePoint{{SpaceID: endpoint.KeySpaceRawKVDefault}}) } @@ -131,12 +135,12 @@ func (s *testStorageGCSuite) TestLoadServiceSafePoint(c *C) { for _, spaceID := range spaceIDs { for _, serviceSafePoint := range serviceSafePoints { - c.Assert(storage.SaveServiceSafePoint(spaceID, serviceSafePoint), IsNil) + c.Assert(storage.SaveServiceSafePointByKeySpace(spaceID, serviceSafePoint), IsNil) } } for _, spaceID := range spaceIDs { for _, serviceSafePoint := range serviceSafePoints { - key := endpoint.ServiceSafePointPath(spaceID, serviceSafePoint.ServiceID) + key := endpoint.KeySpaceServiceSafePointPath(spaceID, serviceSafePoint.ServiceID) value, err := storage.Load(key) c.Assert(err, IsNil) ssp := &endpoint.ServiceSafePoint{} @@ -163,21 +167,21 @@ func (s *testStorageGCSuite) TestRemoveServiceSafePoint(c *C) { // save service safe points for _, spaceID := range spaceIDs { for _, serviceSafePoint := range serviceSafePoints { - c.Assert(storage.SaveServiceSafePoint(spaceID, serviceSafePoint), IsNil) + c.Assert(storage.SaveServiceSafePointByKeySpace(spaceID, serviceSafePoint), IsNil) } } // remove service safe points for _, spaceID := range spaceIDs { for _, serviceSafePoint := range serviceSafePoints { - c.Assert(storage.RemoveServiceSafePoint(spaceID, serviceSafePoint.ServiceID), IsNil) + c.Assert(storage.RemoveServiceSafePointByKeySpace(spaceID, serviceSafePoint.ServiceID), IsNil) } } // check that service safe points are empty for _, spaceID := range spaceIDs { for _, serviceSafePoint := range serviceSafePoints { - safepoint, err := storage.LoadServiceSafePoint(spaceID, serviceSafePoint.ServiceID) + safepoint, err := storage.LoadServiceSafePointByKeySpace(spaceID, serviceSafePoint.ServiceID) c.Assert(err, IsNil) c.Assert(safepoint, IsNil) } @@ -199,26 +203,26 @@ func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { testKeySpace := "test" for _, serviceSafePoint := range serviceSafePoints { - c.Assert(storage.SaveServiceSafePoint(testKeySpace, serviceSafePoint), IsNil) + c.Assert(storage.SaveServiceSafePointByKeySpace(testKeySpace, serviceSafePoint), IsNil) } - minSafePoint, err := storage.LoadMinServiceSafePoint(testKeySpace, currentTime) + minSafePoint, err := storage.LoadMinServiceSafePointByKeySpace(testKeySpace, currentTime) c.Assert(err, IsNil) c.Assert(minSafePoint, DeepEquals, serviceSafePoints[0]) // this should remove safePoint with ServiceID 0 due to expiration // and find the safePoint with ServiceID 1 - minSafePoint2, err := storage.LoadMinServiceSafePoint(testKeySpace, currentTime.Add(150*time.Second)) + minSafePoint2, err := storage.LoadMinServiceSafePointByKeySpace(testKeySpace, currentTime.Add(150*time.Second)) c.Assert(err, IsNil) c.Assert(minSafePoint2, DeepEquals, serviceSafePoints[1]) // verify that service safe point with ServiceID 0 has been removed - ssp, err := storage.LoadServiceSafePoint(testKeySpace, "0") + ssp, err := storage.LoadServiceSafePointByKeySpace(testKeySpace, "0") c.Assert(err, IsNil) c.Assert(ssp, IsNil) // this should remove all service safe points // and return nil - ssp, err = storage.LoadMinServiceSafePoint(testKeySpace, currentTime.Add(500*time.Second)) + ssp, err = storage.LoadMinServiceSafePointByKeySpace(testKeySpace, currentTime.Add(500*time.Second)) c.Assert(err, IsNil) c.Assert(ssp, IsNil) } From 7ff125ee78184c8344230d10bca651e235ad43d3 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 13 May 2022 10:46:18 +0800 Subject: [PATCH 07/34] removed default key spaces Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 32 ++++++++----------------- server/storage/storage_gc_test.go | 20 +++++++++------- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index f17e2a51c9d..3745c217a18 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -24,12 +24,6 @@ import ( "go.etcd.io/etcd/clientv3" ) -// Predefined key spaces. More key spaces would come from "Multi-tenant". -const ( - // KeySpaceRawKVDefault is key space ID for RawKV. - KeySpaceRawKVDefault = "default_rawkv" -) - // KeySpaceGCSafePoint is gcWorker's safepoint for specific key-space type KeySpaceGCSafePoint struct { SpaceID string `json:"space_id"` @@ -46,7 +40,7 @@ type KeySpaceGCSafePointStorage interface { // GC safe point interfaces. SaveGCSafePointByKeySpace(gcSafePoint *KeySpaceGCSafePoint) error LoadGCSafePointByKeySpace(spaceID string) (*KeySpaceGCSafePoint, error) - LoadAllKeySpaceGCSafePoints() ([]*KeySpaceGCSafePoint, error) + LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([]*KeySpaceGCSafePoint, error) } var _ KeySpaceGCSafePointStorage = (*StorageEndpoint)(nil) @@ -150,7 +144,7 @@ func (se *StorageEndpoint) LoadGCSafePointByKeySpace(spaceID string) (*KeySpaceG // LoadAllKeySpaceGCSafePoints returns slice of key-spaces and their corresponding gc safe points. // It also returns spaceID of any default key spaces that do not have a gc safepoint. -func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints() ([]*KeySpaceGCSafePoint, error) { +func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([]*KeySpaceGCSafePoint, error) { prefix := KeySpaceSafePointPrefix() prefixEnd := clientv3.GetPrefixRangeEnd(prefix) suffix := KeySpaceGCSafePointSuffix() @@ -165,22 +159,16 @@ func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints() ([]*KeySpaceGCSafePoint continue } safePoint := &KeySpaceGCSafePoint{} - if err = json.Unmarshal([]byte(values[i]), safePoint); err != nil { - return nil, err + if withGCSafePoint { + if err = json.Unmarshal([]byte(values[i]), safePoint); err != nil { + return nil, err + } + } else { + spaceID := strings.TrimPrefix(keys[i], prefix) + spaceID = strings.TrimSuffix(spaceID, suffix) + safePoint.SpaceID = spaceID } safePoints = append(safePoints, safePoint) } - - // make sure all default key spaces are included in result - defaultKeySpaces := []string{KeySpaceRawKVDefault} - for _, defaultKeySpace := range defaultKeySpaces { - value, err := se.Load(KeySpaceGCSafePointPath(defaultKeySpace)) - if err != nil { - return nil, err - } - if value == "" { - safePoints = append(safePoints, &KeySpaceGCSafePoint{SpaceID: defaultKeySpace}) - } - } return safePoints, nil } diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index 6ec13624e0f..27c6505eccd 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -50,10 +50,6 @@ func testGCSafePoints() []*endpoint.KeySpaceGCSafePoint { SpaceID: "KeySpace5", SafePoint: math.MaxUint64, }, - { - SpaceID: endpoint.KeySpaceRawKVDefault, - SafePoint: 3000, - }, } } @@ -79,7 +75,7 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { err := storage.SaveGCSafePointByKeySpace(safePoint) c.Assert(err, IsNil) } - gcSafePoints, err := storage.LoadAllKeySpaceGCSafePoints() + gcSafePoints, err := storage.LoadAllKeySpaceGCSafePoints(true) c.Assert(err, IsNil) for i, safePoint := range testData { c.Assert(gcSafePoints[i], DeepEquals, safePoint) @@ -105,18 +101,26 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { } // verify that service safe points does not interfere with gc safe points. - gcSafePoints, err = storage.LoadAllKeySpaceGCSafePoints() + gcSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(true) + c.Assert(err, IsNil) + for i, safePoint := range testData { + c.Assert(gcSafePoints[i], DeepEquals, safePoint) + } + + // verify that when withGCSafePoint set to false, returned safe points are 0s. + gcSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(false) c.Assert(err, IsNil) for i, safePoint := range testData { + safePoint.SafePoint = 0 c.Assert(gcSafePoints[i], DeepEquals, safePoint) } } func (s *testStorageGCSuite) TestLoadAllKeySpaces(c *C) { storage := NewStorageWithMemoryBackend() - keySpaces, err := storage.LoadAllKeySpaceGCSafePoints() + keySpaces, err := storage.LoadAllKeySpaceGCSafePoints(true) c.Assert(err, IsNil) - c.Assert(keySpaces, DeepEquals, []*endpoint.KeySpaceGCSafePoint{{SpaceID: endpoint.KeySpaceRawKVDefault}}) + c.Assert(keySpaces, DeepEquals, []*endpoint.KeySpaceGCSafePoint{}) } func (s *testStorageGCSuite) TestLoadServiceSafePoint(c *C) { From 1ebb51d236cd477d95a56df06f954fd3c2ba9b67 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 16 May 2022 14:36:28 +0800 Subject: [PATCH 08/34] renaming, move delete expired safepoints to a goroutine Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 100 +++++++++++------------- 1 file changed, 44 insertions(+), 56 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index 3745c217a18..b7305a20a4d 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -17,6 +17,7 @@ package endpoint import ( "encoding/json" "math" + "strconv" "strings" "time" @@ -24,29 +25,23 @@ import ( "go.etcd.io/etcd/clientv3" ) -// KeySpaceGCSafePoint is gcWorker's safepoint for specific key-space -type KeySpaceGCSafePoint struct { - SpaceID string `json:"space_id"` - SafePoint uint64 `json:"safe_point"` -} - // KeySpaceGCSafePointStorage defines the storage operations on KeySpaces' safe points type KeySpaceGCSafePointStorage interface { // Service safe point interfaces. - SaveServiceSafePointByKeySpace(spaceID string, ssp *ServiceSafePoint) error - LoadServiceSafePointByKeySpace(spaceID, serviceID string) (*ServiceSafePoint, error) - LoadMinServiceSafePointByKeySpace(spaceID string, now time.Time) (*ServiceSafePoint, error) - RemoveServiceSafePointByKeySpace(spaceID, serviceID string) error + SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint) error + LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) + LoadMinServiceSafePoint(spaceID string, now time.Time) (*ServiceSafePoint, error) + RemoveServiceSafePoint(spaceID, serviceID string) error // GC safe point interfaces. - SaveGCSafePointByKeySpace(gcSafePoint *KeySpaceGCSafePoint) error - LoadGCSafePointByKeySpace(spaceID string) (*KeySpaceGCSafePoint, error) - LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([]*KeySpaceGCSafePoint, error) + SaveKeySpaceGCSafePoint(spaceID string, safePoint uint64) error + LoadKeySpaceGCSafePoint(spaceID string) (uint64, error) + LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([]string, []uint64, error) } var _ KeySpaceGCSafePointStorage = (*StorageEndpoint)(nil) -// SaveServiceSafePointByKeySpace saves service safe point under given key-space. -func (se *StorageEndpoint) SaveServiceSafePointByKeySpace(spaceID string, ssp *ServiceSafePoint) error { +// SaveServiceSafePoint saves service safe point under given key-space. +func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint) error { if ssp.ServiceID == "" { return errors.New("service id of service safepoint cannot be empty") } @@ -58,9 +53,9 @@ func (se *StorageEndpoint) SaveServiceSafePointByKeySpace(spaceID string, ssp *S return se.Save(key, string(value)) } -// LoadServiceSafePointByKeySpace reads ServiceSafePoint for the given key-space ID and service name. +// LoadServiceSafePoint reads ServiceSafePoint for the given key-space ID and service name. // Return nil if no safepoint not exist. -func (se *StorageEndpoint) LoadServiceSafePointByKeySpace(spaceID, serviceID string) (*ServiceSafePoint, error) { +func (se *StorageEndpoint) LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) { value, err := se.Load(KeySpaceServiceSafePointPath(spaceID, serviceID)) if err != nil || value == "" { return nil, err @@ -72,10 +67,10 @@ func (se *StorageEndpoint) LoadServiceSafePointByKeySpace(spaceID, serviceID str return ssp, nil } -// LoadMinServiceSafePointByKeySpace returns the minimum safepoint for the given key-space. +// LoadMinServiceSafePoint returns the minimum safepoint for the given key-space. // Note that gc worker safe point are store separately. // If no service safe point exist for the given key-space or all the service safe points just expired, return nil. -func (se *StorageEndpoint) LoadMinServiceSafePointByKeySpace(spaceID string, now time.Time) (*ServiceSafePoint, error) { +func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string, now time.Time) (*ServiceSafePoint, error) { prefix := KeySpaceServiceSafePointPrefix(spaceID) prefixEnd := clientv3.GetPrefixRangeEnd(prefix) keys, values, err := se.LoadRange(prefix, prefixEnd, 0) @@ -92,13 +87,11 @@ func (se *StorageEndpoint) LoadMinServiceSafePointByKeySpace(spaceID string, now // remove expired safe points. if ssp.ExpiredAt < now.Unix() { - err = se.Remove(key) - if err != nil { - return nil, err - } + go func(key string) { + se.Remove(key) + }(key) continue } - if ssp.SafePoint < min.SafePoint { min = ssp } @@ -113,62 +106,57 @@ func (se *StorageEndpoint) LoadMinServiceSafePointByKeySpace(spaceID string, now return min, nil } -// RemoveServiceSafePointByKeySpace removes GCSafePoint for the given key-space. -func (se *StorageEndpoint) RemoveServiceSafePointByKeySpace(spaceID, serviceID string) error { +// RemoveServiceSafePoint removes target ServiceSafePoint +func (se *StorageEndpoint) RemoveServiceSafePoint(spaceID, serviceID string) error { key := KeySpaceServiceSafePointPath(spaceID, serviceID) return se.Remove(key) } -// SaveGCSafePointByKeySpace saves GCSafePoint to the given key-space. -func (se *StorageEndpoint) SaveGCSafePointByKeySpace(gcSafePoint *KeySpaceGCSafePoint) error { - safePoint, err := json.Marshal(gcSafePoint) - if err != nil { - return err - } - return se.Save(KeySpaceGCSafePointPath(gcSafePoint.SpaceID), string(safePoint)) +// SaveKeySpaceGCSafePoint saves GCSafePoint to the given key-space. +func (se *StorageEndpoint) SaveKeySpaceGCSafePoint(spaceID string, safePoint uint64) error { + value := strconv.FormatUint(safePoint, 16) + return se.Save(KeySpaceGCSafePointPath(spaceID), value) } -// LoadGCSafePointByKeySpace reads GCSafePoint for the given key-space. -// return nil if safepoint not exist. -func (se *StorageEndpoint) LoadGCSafePointByKeySpace(spaceID string) (*KeySpaceGCSafePoint, error) { +// LoadKeySpaceGCSafePoint reads GCSafePoint for the given key-space. +// Returns 0 if target safepoint not exist. +func (se *StorageEndpoint) LoadKeySpaceGCSafePoint(spaceID string) (uint64, error) { value, err := se.Load(KeySpaceGCSafePointPath(spaceID)) if err != nil || value == "" { - return nil, err - } - gcSafePoint := &KeySpaceGCSafePoint{} - if err := json.Unmarshal([]byte(value), gcSafePoint); err != nil { - return nil, err + return 0, err } - return gcSafePoint, nil + safePoint, err := strconv.ParseUint(value, 16, 64) + return safePoint, nil } // LoadAllKeySpaceGCSafePoints returns slice of key-spaces and their corresponding gc safe points. -// It also returns spaceID of any default key spaces that do not have a gc safepoint. -func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([]*KeySpaceGCSafePoint, error) { +// If withGCSafePoint set to false, returned safePoints slice will be empty. +func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([]string, []uint64, error) { prefix := KeySpaceSafePointPrefix() prefixEnd := clientv3.GetPrefixRangeEnd(prefix) suffix := KeySpaceGCSafePointSuffix() keys, values, err := se.LoadRange(prefix, prefixEnd, 0) if err != nil { - return nil, err + return nil, nil, err } - safePoints := make([]*KeySpaceGCSafePoint, 0, len(values)) + spaceIDs := make([]string, 0, len(values)) + safePoints := make([]uint64, 0, len(values)) for i := range keys { - // skip non gc safe points + // skip service safe points if !strings.HasSuffix(keys[i], suffix) { continue } - safePoint := &KeySpaceGCSafePoint{} + spaceID := strings.TrimPrefix(keys[i], prefix) + spaceID = strings.TrimSuffix(spaceID, suffix) + spaceIDs = append(spaceIDs, spaceID) + if withGCSafePoint { - if err = json.Unmarshal([]byte(values[i]), safePoint); err != nil { - return nil, err + value, err := strconv.ParseUint(values[i], 16, 64) + if err != nil { + return nil, nil, err } - } else { - spaceID := strings.TrimPrefix(keys[i], prefix) - spaceID = strings.TrimSuffix(spaceID, suffix) - safePoint.SpaceID = spaceID + safePoints = append(safePoints, value) } - safePoints = append(safePoints, safePoint) } - return safePoints, nil + return spaceIDs, safePoints, nil } From 312704a31d8ac58b438eeb9ad9e83c410b650b2b Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 16 May 2022 15:31:14 +0800 Subject: [PATCH 09/34] update tests Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/storage_gc_test.go | 278 ++++++++++++++---------------- 1 file changed, 128 insertions(+), 150 deletions(-) diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index 27c6505eccd..555271b086b 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -15,7 +15,6 @@ package storage import ( - "encoding/json" "math" "time" @@ -28,167 +27,61 @@ var _ = Suite(&testStorageGCSuite{}) type testStorageGCSuite struct { } -func testGCSafePoints() []*endpoint.KeySpaceGCSafePoint { - return []*endpoint.KeySpaceGCSafePoint{ - { - SpaceID: "KeySpace1", - SafePoint: 0, - }, - { - SpaceID: "KeySpace2", - SafePoint: 1, - }, - { - SpaceID: "KeySpace3", - SafePoint: 4396, - }, - { - SpaceID: "KeySpace4", - SafePoint: 23333333333, - }, - { - SpaceID: "KeySpace5", - SafePoint: math.MaxUint64, - }, - } -} - -func (s *testStorageGCSuite) TestLoadGCSafePointByKeySpace(c *C) { - storage := NewStorageWithMemoryBackend() - testData := testGCSafePoints() - r, e := storage.LoadGCSafePointByKeySpace("testKeySpace") - c.Assert(r, IsNil) - c.Assert(e, IsNil) - for _, safePoint := range testData { - err := storage.SaveGCSafePointByKeySpace(safePoint) - c.Assert(err, IsNil) - loaded, err := storage.LoadGCSafePointByKeySpace(safePoint.SpaceID) - c.Assert(err, IsNil) - c.Assert(safePoint, DeepEquals, loaded) - } -} - -func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { - storage := NewStorageWithMemoryBackend() - testData := testGCSafePoints() - for _, safePoint := range testData { - err := storage.SaveGCSafePointByKeySpace(safePoint) - c.Assert(err, IsNil) - } - gcSafePoints, err := storage.LoadAllKeySpaceGCSafePoints(true) - c.Assert(err, IsNil) - for i, safePoint := range testData { - c.Assert(gcSafePoints[i], DeepEquals, safePoint) - } - - // saving some service safe points. - expireAt := time.Now().Add(100 * time.Second).Unix() - serviceSafePoints := []*endpoint.ServiceSafePoint{ - {ServiceID: "1", ExpiredAt: expireAt, SafePoint: 1}, - {ServiceID: "2", ExpiredAt: expireAt, SafePoint: 2}, - {ServiceID: "3", ExpiredAt: expireAt, SafePoint: 3}, - } +func testGCSafePoints() ([]string, []uint64) { spaceIDs := []string{ "keySpace1", "keySpace2", "keySpace3", + "keySpace4", + "keySpace5", } - - for _, spaceID := range spaceIDs { - for _, serviceSafePoint := range serviceSafePoints { - c.Assert(storage.SaveServiceSafePointByKeySpace(spaceID, serviceSafePoint), IsNil) - } - } - - // verify that service safe points does not interfere with gc safe points. - gcSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(true) - c.Assert(err, IsNil) - for i, safePoint := range testData { - c.Assert(gcSafePoints[i], DeepEquals, safePoint) - } - - // verify that when withGCSafePoint set to false, returned safe points are 0s. - gcSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(false) - c.Assert(err, IsNil) - for i, safePoint := range testData { - safePoint.SafePoint = 0 - c.Assert(gcSafePoints[i], DeepEquals, safePoint) + safePoints := []uint64{ + 0, + 1, + 4396, + 23333333333, + math.MaxUint64, } + return spaceIDs, safePoints } -func (s *testStorageGCSuite) TestLoadAllKeySpaces(c *C) { - storage := NewStorageWithMemoryBackend() - keySpaces, err := storage.LoadAllKeySpaceGCSafePoints(true) - c.Assert(err, IsNil) - c.Assert(keySpaces, DeepEquals, []*endpoint.KeySpaceGCSafePoint{}) -} - -func (s *testStorageGCSuite) TestLoadServiceSafePoint(c *C) { - storage := NewStorageWithMemoryBackend() - expireAt := time.Now().Add(100 * time.Second).Unix() - serviceSafePoints := []*endpoint.ServiceSafePoint{ - {ServiceID: "1", ExpiredAt: expireAt, SafePoint: 1}, - {ServiceID: "2", ExpiredAt: expireAt, SafePoint: 2}, - {ServiceID: "3", ExpiredAt: expireAt, SafePoint: 3}, - } +func testServiceSafePoints() ([]string, []*endpoint.ServiceSafePoint) { spaceIDs := []string{ "keySpace1", + "keySpace1", + "keySpace1", + "keySpace2", + "keySpace2", "keySpace2", "keySpace3", + "keySpace3", + "keySpace3", } - - for _, spaceID := range spaceIDs { - for _, serviceSafePoint := range serviceSafePoints { - c.Assert(storage.SaveServiceSafePointByKeySpace(spaceID, serviceSafePoint), IsNil) - } - } - for _, spaceID := range spaceIDs { - for _, serviceSafePoint := range serviceSafePoints { - key := endpoint.KeySpaceServiceSafePointPath(spaceID, serviceSafePoint.ServiceID) - value, err := storage.Load(key) - c.Assert(err, IsNil) - ssp := &endpoint.ServiceSafePoint{} - c.Assert(json.Unmarshal([]byte(value), ssp), IsNil) - c.Assert(ssp, DeepEquals, serviceSafePoint) - } - } -} - -func (s *testStorageGCSuite) TestRemoveServiceSafePoint(c *C) { - storage := NewStorageWithMemoryBackend() expireAt := time.Now().Add(100 * time.Second).Unix() - serviceSafePoints := []*endpoint.ServiceSafePoint{ - {ServiceID: "1", ExpiredAt: expireAt, SafePoint: 1}, - {ServiceID: "2", ExpiredAt: expireAt, SafePoint: 2}, - {ServiceID: "3", ExpiredAt: expireAt, SafePoint: 3}, - } - spaceIDs := []string{ - "keySpace1", - "keySpace2", - "keySpace3", - } - // save service safe points - for _, spaceID := range spaceIDs { - for _, serviceSafePoint := range serviceSafePoints { - c.Assert(storage.SaveServiceSafePointByKeySpace(spaceID, serviceSafePoint), IsNil) - } - } + {ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, + {ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, + {ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, + {ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, + {ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, + {ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, + {ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, + {ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, + {ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, + } + return spaceIDs, serviceSafePoints +} - // remove service safe points - for _, spaceID := range spaceIDs { - for _, serviceSafePoint := range serviceSafePoints { - c.Assert(storage.RemoveServiceSafePointByKeySpace(spaceID, serviceSafePoint.ServiceID), IsNil) - } +func (s *testStorageGCSuite) TestSaveLoadServiceSafePoint(c *C) { + storage := NewStorageWithMemoryBackend() + testSpaceID, testSafePoints := testServiceSafePoints() + for i := range testSpaceID { + c.Assert(storage.SaveServiceSafePoint(testSpaceID[i], testSafePoints[i]), IsNil) } - - // check that service safe points are empty - for _, spaceID := range spaceIDs { - for _, serviceSafePoint := range serviceSafePoints { - safepoint, err := storage.LoadServiceSafePointByKeySpace(spaceID, serviceSafePoint.ServiceID) - c.Assert(err, IsNil) - c.Assert(safepoint, IsNil) - } + for i := range testSpaceID { + loadedSafePoint, err := storage.LoadServiceSafePoint(testSpaceID[i], testSafePoints[i].ServiceID) + c.Assert(err, IsNil) + c.Assert(loadedSafePoint, DeepEquals, testSafePoints[i]) } } @@ -207,26 +100,111 @@ func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { testKeySpace := "test" for _, serviceSafePoint := range serviceSafePoints { - c.Assert(storage.SaveServiceSafePointByKeySpace(testKeySpace, serviceSafePoint), IsNil) + c.Assert(storage.SaveServiceSafePoint(testKeySpace, serviceSafePoint), IsNil) } - minSafePoint, err := storage.LoadMinServiceSafePointByKeySpace(testKeySpace, currentTime) + minSafePoint, err := storage.LoadMinServiceSafePoint(testKeySpace, currentTime) c.Assert(err, IsNil) c.Assert(minSafePoint, DeepEquals, serviceSafePoints[0]) // this should remove safePoint with ServiceID 0 due to expiration // and find the safePoint with ServiceID 1 - minSafePoint2, err := storage.LoadMinServiceSafePointByKeySpace(testKeySpace, currentTime.Add(150*time.Second)) + minSafePoint2, err := storage.LoadMinServiceSafePoint(testKeySpace, currentTime.Add(150*time.Second)) c.Assert(err, IsNil) c.Assert(minSafePoint2, DeepEquals, serviceSafePoints[1]) // verify that service safe point with ServiceID 0 has been removed - ssp, err := storage.LoadServiceSafePointByKeySpace(testKeySpace, "0") + // removal might take some time + time.Sleep(time.Millisecond * 100) + ssp, err := storage.LoadServiceSafePoint(testKeySpace, "0") c.Assert(err, IsNil) c.Assert(ssp, IsNil) // this should remove all service safe points // and return nil - ssp, err = storage.LoadMinServiceSafePointByKeySpace(testKeySpace, currentTime.Add(500*time.Second)) + ssp, err = storage.LoadMinServiceSafePoint(testKeySpace, currentTime.Add(500*time.Second)) c.Assert(err, IsNil) c.Assert(ssp, IsNil) } + +func (s *testStorageGCSuite) TestRemoveServiceSafePoint(c *C) { + storage := NewStorageWithMemoryBackend() + testSpaceID, testSafePoints := testServiceSafePoints() + // save service safe points + for i := range testSpaceID { + c.Assert(storage.SaveServiceSafePoint(testSpaceID[i], testSafePoints[i]), IsNil) + } + // remove saved service safe points + for i := range testSpaceID { + c.Assert(storage.RemoveServiceSafePoint(testSpaceID[i], testSafePoints[i].ServiceID), IsNil) + } + // check that service safe points are empty + for i := range testSpaceID { + loadedSafePoint, err := storage.LoadServiceSafePoint(testSpaceID[i], testSafePoints[i].ServiceID) + c.Assert(err, IsNil) + c.Assert(loadedSafePoint, IsNil) + } +} + +func (s *testStorageGCSuite) TestSaveLoadGCSafePoint(c *C) { + storage := NewStorageWithMemoryBackend() + testSpaceIDs, testSafePoints := testGCSafePoints() + for i := range testSpaceIDs { + testSpaceID := testSpaceIDs[i] + testSafePoint := testSafePoints[i] + err := storage.SaveKeySpaceGCSafePoint(testSpaceID, testSafePoint) + c.Assert(err, IsNil) + loaded, err := storage.LoadKeySpaceGCSafePoint(testSpaceID) + c.Assert(err, IsNil) + c.Assert(loaded, Equals, testSafePoint) + } +} + +func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { + storage := NewStorageWithMemoryBackend() + testSpaceIDs, testSafePoints := testGCSafePoints() + for i := range testSpaceIDs { + err := storage.SaveKeySpaceGCSafePoint(testSpaceIDs[i], testSafePoints[i]) + c.Assert(err, IsNil) + } + loadedSpaceIDs, loadedSafePoints, err := storage.LoadAllKeySpaceGCSafePoints(true) + c.Assert(err, IsNil) + c.Assert(loadedSpaceIDs, DeepEquals, testSpaceIDs) + c.Assert(loadedSafePoints, DeepEquals, testSafePoints) + + // saving some service safe points. + spaceIDs, safePoints := testServiceSafePoints() + for i := range spaceIDs { + c.Assert(storage.SaveServiceSafePoint(spaceIDs[i], safePoints[i]), IsNil) + } + + // verify that service safe points do not interfere with gc safe points. + loadedSpaceIDs, loadedSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(true) + c.Assert(err, IsNil) + c.Assert(loadedSpaceIDs, DeepEquals, testSpaceIDs) + c.Assert(loadedSafePoints, DeepEquals, testSafePoints) + + // verify that when withGCSafePoint set to false, returned safePoints slice is empty. + _, loadedSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(false) + c.Assert(err, IsNil) + c.Assert(loadedSafePoints, HasLen, 0) +} + +func (s *testStorageGCSuite) TestLoadEmpty(c *C) { + storage := NewStorageWithMemoryBackend() + + // loading non-existing GC safepoint should return 0 + gcSafePoint, err := storage.LoadKeySpaceGCSafePoint("testKeySpace") + c.Assert(err, IsNil) + c.Assert(gcSafePoint, Equals, uint64(0)) + + // loading non-existing service safepoint should return nil + serviceSafePoint, err := storage.LoadServiceSafePoint("testKeySpace", "testService") + c.Assert(err, IsNil) + c.Assert(serviceSafePoint, IsNil) + + // loading empty key spaces should return empty slices + spaceIDs, safePoints, err := storage.LoadAllKeySpaceGCSafePoints(true) + c.Assert(err, IsNil) + c.Assert(spaceIDs, HasLen, 0) + c.Assert(safePoints, HasLen, 0) +} From 766b344bf014ec4fc00b73dd483b7df9d1f790d1 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 16 May 2022 15:32:49 +0800 Subject: [PATCH 10/34] lint Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index b7305a20a4d..d0c898e2ef5 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -126,6 +126,9 @@ func (se *StorageEndpoint) LoadKeySpaceGCSafePoint(spaceID string) (uint64, erro return 0, err } safePoint, err := strconv.ParseUint(value, 16, 64) + if err != nil { + return 0, err + } return safePoint, nil } From de77dfcb86eb60fec0b2ed8685afc4e98cf25e1a Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 16 May 2022 16:45:06 +0800 Subject: [PATCH 11/34] added back KeySpaceGCSafePoint Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 32 +++++++++++++++---------- server/storage/storage_gc_test.go | 28 +++++++++++++--------- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index d0c898e2ef5..1aae7b25b3f 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -25,6 +25,12 @@ import ( "go.etcd.io/etcd/clientv3" ) +// KeySpaceGCSafePoint is gcWorker's safepoint for specific key-space +type KeySpaceGCSafePoint struct { + SpaceID string `json:"space_id"` + SafePoint uint64 `json:"safe_point"` +} + // KeySpaceGCSafePointStorage defines the storage operations on KeySpaces' safe points type KeySpaceGCSafePointStorage interface { // Service safe point interfaces. @@ -35,7 +41,7 @@ type KeySpaceGCSafePointStorage interface { // GC safe point interfaces. SaveKeySpaceGCSafePoint(spaceID string, safePoint uint64) error LoadKeySpaceGCSafePoint(spaceID string) (uint64, error) - LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([]string, []uint64, error) + LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([]*KeySpaceGCSafePoint, error) } var _ KeySpaceGCSafePointStorage = (*StorageEndpoint)(nil) @@ -132,34 +138,34 @@ func (se *StorageEndpoint) LoadKeySpaceGCSafePoint(spaceID string) (uint64, erro return safePoint, nil } -// LoadAllKeySpaceGCSafePoints returns slice of key-spaces and their corresponding gc safe points. -// If withGCSafePoint set to false, returned safePoints slice will be empty. -func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([]string, []uint64, error) { +// LoadAllKeySpaceGCSafePoints returns slice of KeySpaceGCSafePoint. +// If withGCSafePoint set to false, returned safePoints will be 0. +func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([]*KeySpaceGCSafePoint, error) { prefix := KeySpaceSafePointPrefix() prefixEnd := clientv3.GetPrefixRangeEnd(prefix) suffix := KeySpaceGCSafePointSuffix() keys, values, err := se.LoadRange(prefix, prefixEnd, 0) if err != nil { - return nil, nil, err + return nil, err } - spaceIDs := make([]string, 0, len(values)) - safePoints := make([]uint64, 0, len(values)) + safePoints := make([]*KeySpaceGCSafePoint, 0, len(values)) for i := range keys { - // skip service safe points + // skip non gc safe points if !strings.HasSuffix(keys[i], suffix) { continue } + safePoint := &KeySpaceGCSafePoint{} spaceID := strings.TrimPrefix(keys[i], prefix) spaceID = strings.TrimSuffix(spaceID, suffix) - spaceIDs = append(spaceIDs, spaceID) - + safePoint.SpaceID = spaceID if withGCSafePoint { value, err := strconv.ParseUint(values[i], 16, 64) if err != nil { - return nil, nil, err + return nil, err } - safePoints = append(safePoints, value) + safePoint.SafePoint = value } + safePoints = append(safePoints, safePoint) } - return spaceIDs, safePoints, nil + return safePoints, nil } diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index 555271b086b..5001b65c1bf 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -166,10 +166,12 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { err := storage.SaveKeySpaceGCSafePoint(testSpaceIDs[i], testSafePoints[i]) c.Assert(err, IsNil) } - loadedSpaceIDs, loadedSafePoints, err := storage.LoadAllKeySpaceGCSafePoints(true) + loadedSafePoints, err := storage.LoadAllKeySpaceGCSafePoints(true) c.Assert(err, IsNil) - c.Assert(loadedSpaceIDs, DeepEquals, testSpaceIDs) - c.Assert(loadedSafePoints, DeepEquals, testSafePoints) + for i := range loadedSafePoints { + c.Assert(loadedSafePoints[i].SpaceID, Equals, testSpaceIDs[i]) + c.Assert(loadedSafePoints[i].SafePoint, Equals, testSafePoints[i]) + } // saving some service safe points. spaceIDs, safePoints := testServiceSafePoints() @@ -178,15 +180,20 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { } // verify that service safe points do not interfere with gc safe points. - loadedSpaceIDs, loadedSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(true) + loadedSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(true) c.Assert(err, IsNil) - c.Assert(loadedSpaceIDs, DeepEquals, testSpaceIDs) - c.Assert(loadedSafePoints, DeepEquals, testSafePoints) + for i := range loadedSafePoints { + c.Assert(loadedSafePoints[i].SpaceID, Equals, testSpaceIDs[i]) + c.Assert(loadedSafePoints[i].SafePoint, Equals, testSafePoints[i]) + } - // verify that when withGCSafePoint set to false, returned safePoints slice is empty. - _, loadedSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(false) + // verify that when withGCSafePoint set to false, returned safePoints is 0 + loadedSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(false) c.Assert(err, IsNil) - c.Assert(loadedSafePoints, HasLen, 0) + for i := range loadedSafePoints { + c.Assert(loadedSafePoints[i].SpaceID, Equals, testSpaceIDs[i]) + c.Assert(loadedSafePoints[i].SafePoint, Equals, uint64(0)) + } } func (s *testStorageGCSuite) TestLoadEmpty(c *C) { @@ -203,8 +210,7 @@ func (s *testStorageGCSuite) TestLoadEmpty(c *C) { c.Assert(serviceSafePoint, IsNil) // loading empty key spaces should return empty slices - spaceIDs, safePoints, err := storage.LoadAllKeySpaceGCSafePoints(true) + safePoints, err := storage.LoadAllKeySpaceGCSafePoints(true) c.Assert(err, IsNil) - c.Assert(spaceIDs, HasLen, 0) c.Assert(safePoints, HasLen, 0) } From d6eda9303f11611e1774edeb4d40b4b109accdad Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 16 May 2022 16:51:52 +0800 Subject: [PATCH 12/34] remove expired all at once Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index 1aae7b25b3f..5c370c05467 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -83,26 +83,29 @@ func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string, now time.Time if err != nil { return nil, err } - min := &ServiceSafePoint{SafePoint: math.MaxUint64} + expiredKeys := make([]string, 0) for i, key := range keys { ssp := &ServiceSafePoint{} if err = json.Unmarshal([]byte(values[i]), ssp); err != nil { return nil, err } - // remove expired safe points. + // gather expired keys if ssp.ExpiredAt < now.Unix() { - go func(key string) { - se.Remove(key) - }(key) + expiredKeys = append(expiredKeys, key) continue } if ssp.SafePoint < min.SafePoint { min = ssp } } - + // remove expired keys + go func() { + for _, key := range expiredKeys { + se.Remove(key) + } + }() if min.SafePoint == math.MaxUint64 { // no service safe point or all of them are expired. return nil, nil From cbe1ed07c1592d917e60188019d532f4731b59d1 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 16 May 2022 17:43:01 +0800 Subject: [PATCH 13/34] address comments Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index 5c370c05467..1b43bd127fb 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -28,7 +28,7 @@ import ( // KeySpaceGCSafePoint is gcWorker's safepoint for specific key-space type KeySpaceGCSafePoint struct { SpaceID string `json:"space_id"` - SafePoint uint64 `json:"safe_point"` + SafePoint uint64 `json:"safe_point,omitempty"` } // KeySpaceGCSafePointStorage defines the storage operations on KeySpaces' safe points @@ -60,9 +60,10 @@ func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafe } // LoadServiceSafePoint reads ServiceSafePoint for the given key-space ID and service name. -// Return nil if no safepoint not exist. +// Return nil if no safepoint exist for given service or just expired. func (se *StorageEndpoint) LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) { - value, err := se.Load(KeySpaceServiceSafePointPath(spaceID, serviceID)) + key := KeySpaceServiceSafePointPath(spaceID, serviceID) + value, err := se.Load(key) if err != nil || value == "" { return nil, err } @@ -70,6 +71,12 @@ func (se *StorageEndpoint) LoadServiceSafePoint(spaceID, serviceID string) (*Ser if err := json.Unmarshal([]byte(value), ssp); err != nil { return nil, err } + if ssp.ExpiredAt < time.Now().Unix() { + go func() { + se.Remove(key) + }() + return nil, nil + } return ssp, nil } From 88610a052b7e62025dca1c11fa4fce9e99efa0a4 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Mon, 16 May 2022 19:47:50 +0800 Subject: [PATCH 14/34] move sleep to failpoint Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 4 ++++ server/storage/storage_gc_test.go | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index 1b43bd127fb..ee34849e5f6 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -22,6 +22,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "go.etcd.io/etcd/clientv3" ) @@ -113,6 +114,9 @@ func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string, now time.Time se.Remove(key) } }() + failpoint.Inject("customTimeout", func() { + time.Sleep(100 * time.Millisecond) + }) if min.SafePoint == math.MaxUint64 { // no service safe point or all of them are expired. return nil, nil diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index 5001b65c1bf..95c67238560 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -19,6 +19,7 @@ import ( "time" . "github.com/pingcap/check" + "github.com/pingcap/failpoint" "github.com/tikv/pd/server/storage/endpoint" ) @@ -86,6 +87,8 @@ func (s *testStorageGCSuite) TestSaveLoadServiceSafePoint(c *C) { } func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { + // enable a custom timeout to accommodate async deletion delay + c.Assert(failpoint.Enable("github.com/tikv/pd/server/storage/endpoint/customTimeout", "return(true)"), IsNil) storage := NewStorageWithMemoryBackend() currentTime := time.Now() expireAt1 := currentTime.Add(100 * time.Second).Unix() @@ -113,8 +116,6 @@ func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { c.Assert(minSafePoint2, DeepEquals, serviceSafePoints[1]) // verify that service safe point with ServiceID 0 has been removed - // removal might take some time - time.Sleep(time.Millisecond * 100) ssp, err := storage.LoadServiceSafePoint(testKeySpace, "0") c.Assert(err, IsNil) c.Assert(ssp, IsNil) @@ -124,6 +125,7 @@ func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { ssp, err = storage.LoadMinServiceSafePoint(testKeySpace, currentTime.Add(500*time.Second)) c.Assert(err, IsNil) c.Assert(ssp, IsNil) + c.Assert(failpoint.Disable("github.com/tikv/pd/server/storage/endpoint/customTimeout"), IsNil) } func (s *testStorageGCSuite) TestRemoveServiceSafePoint(c *C) { From 995033e88d0dea27a99a90a2cef0517a0e885d26 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 17 May 2022 11:13:56 +0800 Subject: [PATCH 15/34] modified failpoint to eliminate sleep Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 12 ++++++++---- server/storage/storage_gc_test.go | 12 +++++------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index ee34849e5f6..42918bfeb2c 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -108,15 +108,19 @@ func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string, now time.Time min = ssp } } - // remove expired keys + // failpoint for immediate removal + failpoint.Inject("removeExpiredKeys", func() { + for _, key := range expiredKeys { + se.Remove(key) + } + expiredKeys = []string{} + }) + // remove expired keys asynchronously go func() { for _, key := range expiredKeys { se.Remove(key) } }() - failpoint.Inject("customTimeout", func() { - time.Sleep(100 * time.Millisecond) - }) if min.SafePoint == math.MaxUint64 { // no service safe point or all of them are expired. return nil, nil diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index 95c67238560..eb0e51c79d0 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -87,8 +87,6 @@ func (s *testStorageGCSuite) TestSaveLoadServiceSafePoint(c *C) { } func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { - // enable a custom timeout to accommodate async deletion delay - c.Assert(failpoint.Enable("github.com/tikv/pd/server/storage/endpoint/customTimeout", "return(true)"), IsNil) storage := NewStorageWithMemoryBackend() currentTime := time.Now() expireAt1 := currentTime.Add(100 * time.Second).Unix() @@ -105,12 +103,13 @@ func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { for _, serviceSafePoint := range serviceSafePoints { c.Assert(storage.SaveServiceSafePoint(testKeySpace, serviceSafePoint), IsNil) } + // enabling failpoint to make expired key removal immediately observable + c.Assert(failpoint.Enable("github.com/tikv/pd/server/storage/endpoint/removeExpiredKeys", "return(true)"), IsNil) minSafePoint, err := storage.LoadMinServiceSafePoint(testKeySpace, currentTime) c.Assert(err, IsNil) c.Assert(minSafePoint, DeepEquals, serviceSafePoints[0]) - // this should remove safePoint with ServiceID 0 due to expiration - // and find the safePoint with ServiceID 1 + // the safePoint with ServiceID 0 should be removed due to expiration minSafePoint2, err := storage.LoadMinServiceSafePoint(testKeySpace, currentTime.Add(150*time.Second)) c.Assert(err, IsNil) c.Assert(minSafePoint2, DeepEquals, serviceSafePoints[1]) @@ -120,12 +119,11 @@ func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { c.Assert(err, IsNil) c.Assert(ssp, IsNil) - // this should remove all service safe points - // and return nil + // all remaining service safePoints should be removed due to expiration ssp, err = storage.LoadMinServiceSafePoint(testKeySpace, currentTime.Add(500*time.Second)) c.Assert(err, IsNil) c.Assert(ssp, IsNil) - c.Assert(failpoint.Disable("github.com/tikv/pd/server/storage/endpoint/customTimeout"), IsNil) + c.Assert(failpoint.Disable("github.com/tikv/pd/server/storage/endpoint/removeExpiredKeys"), IsNil) } func (s *testStorageGCSuite) TestRemoveServiceSafePoint(c *C) { From a825d37f0c61753edfce6dd8c413af2699f76d31 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 18 May 2022 14:11:36 +0800 Subject: [PATCH 16/34] log error when failed to remove expired service safe point Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index 42918bfeb2c..e23197da90e 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -23,7 +23,10 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/errs" "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" ) // KeySpaceGCSafePoint is gcWorker's safepoint for specific key-space @@ -74,7 +77,9 @@ func (se *StorageEndpoint) LoadServiceSafePoint(spaceID, serviceID string) (*Ser } if ssp.ExpiredAt < time.Now().Unix() { go func() { - se.Remove(key) + if err = se.Remove(key); err != nil { + log.Error("remove expired key meet error", zap.String("key", key), errs.ZapError(err)) + } }() return nil, nil } @@ -111,14 +116,18 @@ func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string, now time.Time // failpoint for immediate removal failpoint.Inject("removeExpiredKeys", func() { for _, key := range expiredKeys { - se.Remove(key) + if err = se.Remove(key); err != nil { + log.Error("remove expired key meet error", zap.String("key", key), errs.ZapError(err)) + } } expiredKeys = []string{} }) // remove expired keys asynchronously go func() { for _, key := range expiredKeys { - se.Remove(key) + if err = se.Remove(key); err != nil { + log.Error("remove expired key meet error", zap.String("key", key), errs.ZapError(err)) + } } }() if min.SafePoint == math.MaxUint64 { From 8bef6ee3a50e128ad85bb7c8c1b992e2896b6f48 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 18 May 2022 16:54:46 +0800 Subject: [PATCH 17/34] added load revision Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/kv/etcd_kv.go | 14 ++++++++++---- server/storage/kv/kv.go | 5 +++++ server/storage/kv/levedb_kv.go | 6 ++++++ server/storage/kv/mem_kv.go | 6 ++++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/server/storage/kv/etcd_kv.go b/server/storage/kv/etcd_kv.go index e30b5f7b462..e2232635832 100644 --- a/server/storage/kv/etcd_kv.go +++ b/server/storage/kv/etcd_kv.go @@ -48,18 +48,24 @@ func NewEtcdKVBase(client *clientv3.Client, rootPath string) *etcdKVBase { } func (kv *etcdKVBase) Load(key string) (string, error) { + value, _, err := kv.LoadRevision(key) + return value, err +} + +// LoadRevision gets a value along with revision. +func (kv *etcdKVBase) LoadRevision(key string) (string, int64, error) { key = path.Join(kv.rootPath, key) resp, err := etcdutil.EtcdKVGet(kv.client, key) if err != nil { - return "", err + return "", RevisionUnavailable, err } if n := len(resp.Kvs); n == 0 { - return "", nil + return "", RevisionUnavailable, nil } else if n > 1 { - return "", errs.ErrEtcdKVGetResponse.GenWithStackByArgs(resp.Kvs) + return "", RevisionUnavailable, errs.ErrEtcdKVGetResponse.GenWithStackByArgs(resp.Kvs) } - return string(resp.Kvs[0].Value), nil + return string(resp.Kvs[0].Value), resp.Kvs[0].ModRevision, nil } func (kv *etcdKVBase) LoadRange(key, endKey string, limit int) ([]string, []string, error) { diff --git a/server/storage/kv/kv.go b/server/storage/kv/kv.go index 2f1fa06e144..ed15c7c9b09 100644 --- a/server/storage/kv/kv.go +++ b/server/storage/kv/kv.go @@ -14,10 +14,15 @@ package kv +// RevisionUnavailable is the value of unavailable revision, +// when the kv does not exist (etcd_kv), or is not supported (mem_kv & leveldb_kv). +const RevisionUnavailable = -1 + // Base is an abstract interface for load/save pd cluster data. type Base interface { Load(key string) (string, error) LoadRange(key, endKey string, limit int) (keys []string, values []string, err error) + LoadRevision(key string) (string, int64, error) Save(key, value string) error Remove(key string) error } diff --git a/server/storage/kv/levedb_kv.go b/server/storage/kv/levedb_kv.go index 7f134709bd1..38c086e71ae 100644 --- a/server/storage/kv/levedb_kv.go +++ b/server/storage/kv/levedb_kv.go @@ -49,6 +49,12 @@ func (kv *LevelDBKV) Load(key string) (string, error) { return string(v), err } +// LoadRevision gets a value along with revision. The revision is unavailable for `LevelDBKV`. +func (kv *LevelDBKV) LoadRevision(key string) (string, int64, error) { + value, err := kv.Load(key) + return value, RevisionUnavailable, err +} + // LoadRange gets a range of value for a given key range. func (kv *LevelDBKV) LoadRange(startKey, endKey string, limit int) ([]string, []string, error) { iter := kv.NewIterator(&util.Range{Start: []byte(startKey), Limit: []byte(endKey)}, nil) diff --git a/server/storage/kv/mem_kv.go b/server/storage/kv/mem_kv.go index b74cab84b11..51b4af220f5 100644 --- a/server/storage/kv/mem_kv.go +++ b/server/storage/kv/mem_kv.go @@ -51,6 +51,12 @@ func (kv *memoryKV) Load(key string) (string, error) { return item.(memoryKVItem).value, nil } +// LoadRevision gets a value along with revision. The revision is unavailable for `memoryKV`. +func (kv *memoryKV) LoadRevision(key string) (string, int64, error) { + value, err := kv.Load(key) + return value, RevisionUnavailable, err +} + func (kv *memoryKV) LoadRange(key, endKey string, limit int) ([]string, []string, error) { failpoint.Inject("withRangeLimit", func(val failpoint.Value) { rangeLimit, ok := val.(int) From 89cf4d0424f9b455c684172fd79c7e88e37c863a Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 18 May 2022 22:59:32 +0800 Subject: [PATCH 18/34] added SaveWithTTL to kvs Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/kv/etcd_kv.go | 50 ++++++++++++++++++++++++++++++++++ server/storage/kv/kv.go | 1 + server/storage/kv/levedb_kv.go | 5 ++++ server/storage/kv/mem_kv.go | 5 ++++ 4 files changed, 61 insertions(+) diff --git a/server/storage/kv/etcd_kv.go b/server/storage/kv/etcd_kv.go index e2232635832..3a17497dfdc 100644 --- a/server/storage/kv/etcd_kv.go +++ b/server/storage/kv/etcd_kv.go @@ -109,6 +109,56 @@ func (kv *etcdKVBase) Save(key, value string) error { return nil } +func (kv *etcdKVBase) GrantLease(ttlSeconds int64) (leaseID clientv3.LeaseID, err error) { + start := time.Now() + ctx, cancel := context.WithTimeout(kv.client.Ctx(), requestTimeout) + grantResp, err := kv.client.Grant(ctx, ttlSeconds) + cancel() + if err != nil { + e := errs.ErrEtcdGrantLease.Wrap(err).GenWithStackByCause() + log.Error("grant lease meet error", + zap.Int64("ttl", ttlSeconds), + errs.ZapError(e)) + return 0, e + } + if cost := time.Since(start); cost > slowRequestTime { + log.Warn("lease grants too slow", + zap.Reflect("response", grantResp), + zap.Duration("cost", cost), + errs.ZapError(err)) + } + log.Info("lease granted", + zap.Int64("lease-id", int64(grantResp.ID)), + zap.Int64("lease-timeout", ttlSeconds)) + return grantResp.ID, nil +} + +func (kv *etcdKVBase) SaveWithTTL(key, value string, ttlSeconds int64) error { + + leaseID, err := kv.GrantLease(ttlSeconds) + if err != nil { + return errs.ErrEtcdKVPut.Wrap(err).GenWithStackByCause() + } + + key = path.Join(kv.rootPath, key) + txn := NewSlowLogTxn(kv.client) + resp, err := txn.Then(clientv3.OpPut(key, value, clientv3.WithLease(leaseID))).Commit() + if err != nil { + e := errs.ErrEtcdKVPut.Wrap(err).GenWithStackByCause() + log.Error("save to etcd with lease meet error", + zap.String("key", key), + zap.String("value", value), + zap.Int64("lease-id", int64(leaseID)), + errs.ZapError(e), + ) + return e + } + if !resp.Succeeded { + return errs.ErrEtcdTxnConflict.FastGenByArgs() + } + return nil +} + func (kv *etcdKVBase) Remove(key string) error { key = path.Join(kv.rootPath, key) diff --git a/server/storage/kv/kv.go b/server/storage/kv/kv.go index ed15c7c9b09..f7fd609302a 100644 --- a/server/storage/kv/kv.go +++ b/server/storage/kv/kv.go @@ -24,5 +24,6 @@ type Base interface { LoadRange(key, endKey string, limit int) (keys []string, values []string, err error) LoadRevision(key string) (string, int64, error) Save(key, value string) error + SaveWithTTL(key, value string, ttlSeconds int64) error Remove(key string) error } diff --git a/server/storage/kv/levedb_kv.go b/server/storage/kv/levedb_kv.go index 38c086e71ae..9f2ed2770ab 100644 --- a/server/storage/kv/levedb_kv.go +++ b/server/storage/kv/levedb_kv.go @@ -78,6 +78,11 @@ func (kv *LevelDBKV) Save(key, value string) error { return errors.WithStack(kv.Put([]byte(key), []byte(value), nil)) } +// SaveWithTTL not supported on LevelDBKV +func (kv *LevelDBKV) SaveWithTTL(key, value string, ttlSeconds int64) error { + return errors.New("ttl operation not supported on LevelDBKV") +} + // Remove deletes a key-value pair for a given key. func (kv *LevelDBKV) Remove(key string) error { return errors.WithStack(kv.Delete([]byte(key), nil)) diff --git a/server/storage/kv/mem_kv.go b/server/storage/kv/mem_kv.go index 51b4af220f5..937a8331277 100644 --- a/server/storage/kv/mem_kv.go +++ b/server/storage/kv/mem_kv.go @@ -86,6 +86,11 @@ func (kv *memoryKV) Save(key, value string) error { return nil } +// SaveWithTTL not supported on memoryKV +func (kv *memoryKV) SaveWithTTL(key, value string, ttlSeconds int64) error { + return errors.New("ttl operation not supported on memoryKV") +} + func (kv *memoryKV) Remove(key string) error { kv.Lock() defer kv.Unlock() From b8bc86f421a49045603d76ac2d1482c59dc18c18 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 18 May 2022 23:10:50 +0800 Subject: [PATCH 19/34] updated storage methods to use lease Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 59 +++++-------------------- 1 file changed, 11 insertions(+), 48 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index e23197da90e..74995b0c2db 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -16,17 +16,11 @@ package endpoint import ( "encoding/json" + "github.com/pingcap/errors" + "go.etcd.io/etcd/clientv3" "math" "strconv" "strings" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/pd/pkg/errs" - "go.etcd.io/etcd/clientv3" - "go.uber.org/zap" ) // KeySpaceGCSafePoint is gcWorker's safepoint for specific key-space @@ -38,9 +32,10 @@ type KeySpaceGCSafePoint struct { // KeySpaceGCSafePointStorage defines the storage operations on KeySpaces' safe points type KeySpaceGCSafePointStorage interface { // Service safe point interfaces. - SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint) error + // NOTE: field ServiceSafePoint.ExpiredAt will be ignored, use etcd's lease to manage lifetime instead. + SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint, ttl int64) error LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) - LoadMinServiceSafePoint(spaceID string, now time.Time) (*ServiceSafePoint, error) + LoadMinServiceSafePoint(spaceID string) (*ServiceSafePoint, error) RemoveServiceSafePoint(spaceID, serviceID string) error // GC safe point interfaces. SaveKeySpaceGCSafePoint(spaceID string, safePoint uint64) error @@ -51,7 +46,7 @@ type KeySpaceGCSafePointStorage interface { var _ KeySpaceGCSafePointStorage = (*StorageEndpoint)(nil) // SaveServiceSafePoint saves service safe point under given key-space. -func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint) error { +func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint, ttl int64) error { if ssp.ServiceID == "" { return errors.New("service id of service safepoint cannot be empty") } @@ -60,11 +55,11 @@ func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafe if err != nil { return err } - return se.Save(key, string(value)) + return se.SaveWithTTL(key, string(value), ttl) } // LoadServiceSafePoint reads ServiceSafePoint for the given key-space ID and service name. -// Return nil if no safepoint exist for given service or just expired. +// Return nil if no safepoint exist for given service. func (se *StorageEndpoint) LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) { key := KeySpaceServiceSafePointPath(spaceID, serviceID) value, err := se.Load(key) @@ -75,61 +70,29 @@ func (se *StorageEndpoint) LoadServiceSafePoint(spaceID, serviceID string) (*Ser if err := json.Unmarshal([]byte(value), ssp); err != nil { return nil, err } - if ssp.ExpiredAt < time.Now().Unix() { - go func() { - if err = se.Remove(key); err != nil { - log.Error("remove expired key meet error", zap.String("key", key), errs.ZapError(err)) - } - }() - return nil, nil - } return ssp, nil } // LoadMinServiceSafePoint returns the minimum safepoint for the given key-space. // Note that gc worker safe point are store separately. // If no service safe point exist for the given key-space or all the service safe points just expired, return nil. -func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string, now time.Time) (*ServiceSafePoint, error) { +func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string) (*ServiceSafePoint, error) { prefix := KeySpaceServiceSafePointPrefix(spaceID) prefixEnd := clientv3.GetPrefixRangeEnd(prefix) - keys, values, err := se.LoadRange(prefix, prefixEnd, 0) + _, values, err := se.LoadRange(prefix, prefixEnd, 0) if err != nil { return nil, err } min := &ServiceSafePoint{SafePoint: math.MaxUint64} - expiredKeys := make([]string, 0) - for i, key := range keys { + for i := range values { ssp := &ServiceSafePoint{} if err = json.Unmarshal([]byte(values[i]), ssp); err != nil { return nil, err } - - // gather expired keys - if ssp.ExpiredAt < now.Unix() { - expiredKeys = append(expiredKeys, key) - continue - } if ssp.SafePoint < min.SafePoint { min = ssp } } - // failpoint for immediate removal - failpoint.Inject("removeExpiredKeys", func() { - for _, key := range expiredKeys { - if err = se.Remove(key); err != nil { - log.Error("remove expired key meet error", zap.String("key", key), errs.ZapError(err)) - } - } - expiredKeys = []string{} - }) - // remove expired keys asynchronously - go func() { - for _, key := range expiredKeys { - if err = se.Remove(key); err != nil { - log.Error("remove expired key meet error", zap.String("key", key), errs.ZapError(err)) - } - } - }() if min.SafePoint == math.MaxUint64 { // no service safe point or all of them are expired. return nil, nil From 1eca8d9e1f9586a8bf95c1d4f13f11cfcd8e2d3d Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Thu, 19 May 2022 00:21:25 +0800 Subject: [PATCH 20/34] update tests to use ttl Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/kv/etcd_kv.go | 1 - server/storage/storage_gc_test.go | 146 +++++++++++++++++++++--------- 2 files changed, 103 insertions(+), 44 deletions(-) diff --git a/server/storage/kv/etcd_kv.go b/server/storage/kv/etcd_kv.go index 3a17497dfdc..4bb899917a6 100644 --- a/server/storage/kv/etcd_kv.go +++ b/server/storage/kv/etcd_kv.go @@ -134,7 +134,6 @@ func (kv *etcdKVBase) GrantLease(ttlSeconds int64) (leaseID clientv3.LeaseID, er } func (kv *etcdKVBase) SaveWithTTL(key, value string, ttlSeconds int64) error { - leaseID, err := kv.GrantLease(ttlSeconds) if err != nil { return errs.ErrEtcdKVPut.Wrap(err).GenWithStackByCause() diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index eb0e51c79d0..24ed77be9ec 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -15,17 +15,47 @@ package storage import ( + "fmt" "math" + "net/url" + "os" + "path" + "strconv" "time" . "github.com/pingcap/check" - "github.com/pingcap/failpoint" + "github.com/tikv/pd/pkg/tempurl" "github.com/tikv/pd/server/storage/endpoint" + "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/embed" ) var _ = Suite(&testStorageGCSuite{}) type testStorageGCSuite struct { + cfg *embed.Config + etcd *embed.Etcd + storage Storage +} + +func (s *testStorageGCSuite) SetUpTest(c *C) { + s.cfg = newTestSingleConfig() + s.etcd = mustNewEmbedEtcd(c, s.cfg) + + ep := s.cfg.LCUrls[0].String() + client, err := clientv3.New(clientv3.Config{ + Endpoints: []string{ep}, + }) + c.Assert(err, IsNil) + rootPath := path.Join("/pd", strconv.FormatUint(100, 10)) + s.storage = NewStorageWithEtcdBackend(client, rootPath) +} + +func (s *testStorageGCSuite) TearDownTest(c *C) { + if s.etcd != nil { + s.etcd.Close() + } + c.Assert(cleanConfig(s.cfg), IsNil) } func testGCSafePoints() ([]string, []uint64) { @@ -46,7 +76,7 @@ func testGCSafePoints() ([]string, []uint64) { return spaceIDs, safePoints } -func testServiceSafePoints() ([]string, []*endpoint.ServiceSafePoint) { +func testServiceSafePoints() ([]string, []*endpoint.ServiceSafePoint, []int64) { spaceIDs := []string{ "keySpace1", "keySpace1", @@ -58,26 +88,29 @@ func testServiceSafePoints() ([]string, []*endpoint.ServiceSafePoint) { "keySpace3", "keySpace3", } - expireAt := time.Now().Add(100 * time.Second).Unix() serviceSafePoints := []*endpoint.ServiceSafePoint{ - {ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, - {ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, - {ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, - {ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, - {ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, - {ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, - {ServiceID: "service1", ExpiredAt: expireAt, SafePoint: 1}, - {ServiceID: "service2", ExpiredAt: expireAt, SafePoint: 2}, - {ServiceID: "service3", ExpiredAt: expireAt, SafePoint: 3}, - } - return spaceIDs, serviceSafePoints + {ServiceID: "service1", SafePoint: 1}, + {ServiceID: "service2", SafePoint: 2}, + {ServiceID: "service3", SafePoint: 3}, + {ServiceID: "service1", SafePoint: 1}, + {ServiceID: "service2", SafePoint: 2}, + {ServiceID: "service3", SafePoint: 3}, + {ServiceID: "service1", SafePoint: 1}, + {ServiceID: "service2", SafePoint: 2}, + {ServiceID: "service3", SafePoint: 3}, + } + testTTls := make([]int64, 9) + for i := range testTTls { + testTTls[i] = 10 + } + return spaceIDs, serviceSafePoints, testTTls } func (s *testStorageGCSuite) TestSaveLoadServiceSafePoint(c *C) { - storage := NewStorageWithMemoryBackend() - testSpaceID, testSafePoints := testServiceSafePoints() + storage := s.storage + testSpaceID, testSafePoints, testTTLs := testServiceSafePoints() for i := range testSpaceID { - c.Assert(storage.SaveServiceSafePoint(testSpaceID[i], testSafePoints[i]), IsNil) + c.Assert(storage.SaveServiceSafePoint(testSpaceID[i], testSafePoints[i], testTTLs[i]), IsNil) } for i := range testSpaceID { loadedSafePoint, err := storage.LoadServiceSafePoint(testSpaceID[i], testSafePoints[i].ServiceID) @@ -87,30 +120,25 @@ func (s *testStorageGCSuite) TestSaveLoadServiceSafePoint(c *C) { } func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { - storage := NewStorageWithMemoryBackend() - currentTime := time.Now() - expireAt1 := currentTime.Add(100 * time.Second).Unix() - expireAt2 := currentTime.Add(200 * time.Second).Unix() - expireAt3 := currentTime.Add(300 * time.Second).Unix() - + storage := s.storage + testTTLs := []int64{2, 6} serviceSafePoints := []*endpoint.ServiceSafePoint{ - {ServiceID: "0", ExpiredAt: expireAt1, SafePoint: 100}, - {ServiceID: "1", ExpiredAt: expireAt2, SafePoint: 200}, - {ServiceID: "2", ExpiredAt: expireAt3, SafePoint: 300}, + {ServiceID: "0", SafePoint: 100}, + {ServiceID: "1", SafePoint: 200}, } - testKeySpace := "test" - for _, serviceSafePoint := range serviceSafePoints { - c.Assert(storage.SaveServiceSafePoint(testKeySpace, serviceSafePoint), IsNil) + for i := range serviceSafePoints { + c.Assert(storage.SaveServiceSafePoint(testKeySpace, serviceSafePoints[i], testTTLs[i]), IsNil) } - // enabling failpoint to make expired key removal immediately observable - c.Assert(failpoint.Enable("github.com/tikv/pd/server/storage/endpoint/removeExpiredKeys", "return(true)"), IsNil) - minSafePoint, err := storage.LoadMinServiceSafePoint(testKeySpace, currentTime) + + minSafePoint, err := storage.LoadMinServiceSafePoint(testKeySpace) c.Assert(err, IsNil) c.Assert(minSafePoint, DeepEquals, serviceSafePoints[0]) + time.Sleep(4 * time.Second) // the safePoint with ServiceID 0 should be removed due to expiration - minSafePoint2, err := storage.LoadMinServiceSafePoint(testKeySpace, currentTime.Add(150*time.Second)) + // now min should be safePoint with ServiceID 1 + minSafePoint2, err := storage.LoadMinServiceSafePoint(testKeySpace) c.Assert(err, IsNil) c.Assert(minSafePoint2, DeepEquals, serviceSafePoints[1]) @@ -119,19 +147,19 @@ func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { c.Assert(err, IsNil) c.Assert(ssp, IsNil) + time.Sleep(4 * time.Second) // all remaining service safePoints should be removed due to expiration - ssp, err = storage.LoadMinServiceSafePoint(testKeySpace, currentTime.Add(500*time.Second)) + ssp, err = storage.LoadMinServiceSafePoint(testKeySpace) c.Assert(err, IsNil) c.Assert(ssp, IsNil) - c.Assert(failpoint.Disable("github.com/tikv/pd/server/storage/endpoint/removeExpiredKeys"), IsNil) } func (s *testStorageGCSuite) TestRemoveServiceSafePoint(c *C) { - storage := NewStorageWithMemoryBackend() - testSpaceID, testSafePoints := testServiceSafePoints() + storage := s.storage + testSpaceID, testSafePoints, testTTLs := testServiceSafePoints() // save service safe points for i := range testSpaceID { - c.Assert(storage.SaveServiceSafePoint(testSpaceID[i], testSafePoints[i]), IsNil) + c.Assert(storage.SaveServiceSafePoint(testSpaceID[i], testSafePoints[i], testTTLs[i]), IsNil) } // remove saved service safe points for i := range testSpaceID { @@ -146,7 +174,7 @@ func (s *testStorageGCSuite) TestRemoveServiceSafePoint(c *C) { } func (s *testStorageGCSuite) TestSaveLoadGCSafePoint(c *C) { - storage := NewStorageWithMemoryBackend() + storage := s.storage testSpaceIDs, testSafePoints := testGCSafePoints() for i := range testSpaceIDs { testSpaceID := testSpaceIDs[i] @@ -160,7 +188,7 @@ func (s *testStorageGCSuite) TestSaveLoadGCSafePoint(c *C) { } func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { - storage := NewStorageWithMemoryBackend() + storage := s.storage testSpaceIDs, testSafePoints := testGCSafePoints() for i := range testSpaceIDs { err := storage.SaveKeySpaceGCSafePoint(testSpaceIDs[i], testSafePoints[i]) @@ -174,9 +202,9 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { } // saving some service safe points. - spaceIDs, safePoints := testServiceSafePoints() + spaceIDs, safePoints, TTLs := testServiceSafePoints() for i := range spaceIDs { - c.Assert(storage.SaveServiceSafePoint(spaceIDs[i], safePoints[i]), IsNil) + c.Assert(storage.SaveServiceSafePoint(spaceIDs[i], safePoints[i], TTLs[i]), IsNil) } // verify that service safe points do not interfere with gc safe points. @@ -197,7 +225,7 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { } func (s *testStorageGCSuite) TestLoadEmpty(c *C) { - storage := NewStorageWithMemoryBackend() + storage := s.storage // loading non-existing GC safepoint should return 0 gcSafePoint, err := storage.LoadKeySpaceGCSafePoint("testKeySpace") @@ -214,3 +242,35 @@ func (s *testStorageGCSuite) TestLoadEmpty(c *C) { c.Assert(err, IsNil) c.Assert(safePoints, HasLen, 0) } + +func newTestSingleConfig() *embed.Config { + cfg := embed.NewConfig() + cfg.Name = "test_etcd" + cfg.Dir, _ = os.MkdirTemp("/tmp", "test_etcd") + cfg.WalDir = "" + cfg.Logger = "zap" + cfg.LogOutputs = []string{"stdout"} + + pu, _ := url.Parse(tempurl.Alloc()) + cfg.LPUrls = []url.URL{*pu} + cfg.APUrls = cfg.LPUrls + cu, _ := url.Parse(tempurl.Alloc()) + cfg.LCUrls = []url.URL{*cu} + cfg.ACUrls = cfg.LCUrls + + cfg.StrictReconfigCheck = false + cfg.InitialCluster = fmt.Sprintf("%s=%s", cfg.Name, &cfg.LPUrls[0]) + cfg.ClusterState = embed.ClusterStateFlagNew + return cfg +} + +func cleanConfig(cfg *embed.Config) error { + // Clean data directory + return os.RemoveAll(cfg.Dir) +} + +func mustNewEmbedEtcd(c *C, cfg *embed.Config) *embed.Etcd { + etcd, err := embed.StartEtcd(cfg) + c.Assert(err, IsNil) + return etcd +} From 176061ecdf86c17e89a431538f4437a8224e575d Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 27 May 2022 14:26:40 +0800 Subject: [PATCH 21/34] lint Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- go.mod | 1 + go.sum | 1 + server/storage/endpoint/gc_key_space.go | 5 ----- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 875eb6efe7c..aab95d5d965 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/AlekSi/gocov-xml v1.0.0 github.com/BurntSushi/toml v0.3.1 + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/aws/aws-sdk-go v1.35.3 github.com/axw/gocov v1.0.0 github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 diff --git a/go.sum b/go.sum index 50997691e2a..702a4b7866c 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,7 @@ github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQ github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index c4747bd0ef9..2bb5c9cdc1c 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -19,14 +19,9 @@ import ( "math" "strconv" "strings" - "time" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/pd/pkg/errs" "go.etcd.io/etcd/clientv3" - "go.uber.org/zap" ) // KeySpaceGCSafePoint is gcWorker's safepoint for specific key-space From 5d994ea1a378e0dd6bb56f8dd64c656bf6c63ca5 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 27 May 2022 14:43:12 +0800 Subject: [PATCH 22/34] use etcdutil to save with ttl Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/kv/etcd_kv.go | 70 +++++++++++++----------------------- 1 file changed, 25 insertions(+), 45 deletions(-) diff --git a/server/storage/kv/etcd_kv.go b/server/storage/kv/etcd_kv.go index 4bb899917a6..63594d46ccf 100644 --- a/server/storage/kv/etcd_kv.go +++ b/server/storage/kv/etcd_kv.go @@ -91,67 +91,47 @@ func (kv *etcdKVBase) LoadRange(key, endKey string, limit int) ([]string, []stri return keys, values, nil } -func (kv *etcdKVBase) Save(key, value string) error { - failpoint.Inject("etcdSaveFailed", func() { - failpoint.Return(errors.New("save failed")) - }) +func (kv *etcdKVBase) SaveWithTTL(key, value string, ttlSeconds int64) error { key = path.Join(kv.rootPath, key) - txn := NewSlowLogTxn(kv.client) - resp, err := txn.Then(clientv3.OpPut(key, value)).Commit() - if err != nil { - e := errs.ErrEtcdKVPut.Wrap(err).GenWithStackByCause() - log.Error("save to etcd meet error", zap.String("key", key), zap.String("value", value), errs.ZapError(e)) - return e - } - if !resp.Succeeded { - return errs.ErrEtcdTxnConflict.FastGenByArgs() - } - return nil -} - -func (kv *etcdKVBase) GrantLease(ttlSeconds int64) (leaseID clientv3.LeaseID, err error) { - start := time.Now() + ctx, cancel := context.WithTimeout(kv.client.Ctx(), requestTimeout) - grantResp, err := kv.client.Grant(ctx, ttlSeconds) + start := time.Now() + resp, err := etcdutil.EtcdKVPutWithTTL(ctx, kv.client, key, value, ttlSeconds) cancel() - if err != nil { - e := errs.ErrEtcdGrantLease.Wrap(err).GenWithStackByCause() - log.Error("grant lease meet error", - zap.Int64("ttl", ttlSeconds), - errs.ZapError(e)) - return 0, e - } - if cost := time.Since(start); cost > slowRequestTime { - log.Warn("lease grants too slow", - zap.Reflect("response", grantResp), + + cost := time.Since(start) + if cost > slowRequestTime { + log.Warn("save to etcd with lease runs too slow", + zap.Reflect("response", resp), zap.Duration("cost", cost), errs.ZapError(err)) } - log.Info("lease granted", - zap.Int64("lease-id", int64(grantResp.ID)), - zap.Int64("lease-timeout", ttlSeconds)) - return grantResp.ID, nil -} - -func (kv *etcdKVBase) SaveWithTTL(key, value string, ttlSeconds int64) error { - leaseID, err := kv.GrantLease(ttlSeconds) - if err != nil { - return errs.ErrEtcdKVPut.Wrap(err).GenWithStackByCause() - } - key = path.Join(kv.rootPath, key) - txn := NewSlowLogTxn(kv.client) - resp, err := txn.Then(clientv3.OpPut(key, value, clientv3.WithLease(leaseID))).Commit() if err != nil { e := errs.ErrEtcdKVPut.Wrap(err).GenWithStackByCause() log.Error("save to etcd with lease meet error", zap.String("key", key), zap.String("value", value), - zap.Int64("lease-id", int64(leaseID)), + zap.Int64("ttl-seconds", ttlSeconds), errs.ZapError(e), ) return e } + return nil +} + +func (kv *etcdKVBase) Save(key, value string) error { + failpoint.Inject("etcdSaveFailed", func() { + failpoint.Return(errors.New("save failed")) + }) + key = path.Join(kv.rootPath, key) + txn := NewSlowLogTxn(kv.client) + resp, err := txn.Then(clientv3.OpPut(key, value)).Commit() + if err != nil { + e := errs.ErrEtcdKVPut.Wrap(err).GenWithStackByCause() + log.Error("save to etcd meet error", zap.String("key", key), zap.String("value", value), errs.ZapError(e)) + return e + } if !resp.Succeeded { return errs.ErrEtcdTxnConflict.FastGenByArgs() } From a36305a80076439b00262096586f86906e4d8939 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Fri, 27 May 2022 15:03:27 +0800 Subject: [PATCH 23/34] update go mod Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/kv/etcd_kv.go | 1 - tests/client/go.sum | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/server/storage/kv/etcd_kv.go b/server/storage/kv/etcd_kv.go index 63594d46ccf..7efeb61ae1f 100644 --- a/server/storage/kv/etcd_kv.go +++ b/server/storage/kv/etcd_kv.go @@ -93,7 +93,6 @@ func (kv *etcdKVBase) LoadRange(key, endKey string, limit int) ([]string, []stri func (kv *etcdKVBase) SaveWithTTL(key, value string, ttlSeconds int64) error { key = path.Join(kv.rootPath, key) - ctx, cancel := context.WithTimeout(kv.client.Ctx(), requestTimeout) start := time.Now() resp, err := etcdutil.EtcdKVPutWithTTL(ctx, kv.client, key, value, ttlSeconds) diff --git a/tests/client/go.sum b/tests/client/go.sum index 493b1cd39f4..3f83cb9a19b 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -24,6 +24,7 @@ github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQ github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= From d8e7ab699bd07f721764a75279d7b2236faec4a5 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 31 May 2022 15:58:18 +0800 Subject: [PATCH 24/34] update go mod Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- go.mod | 1 - go.sum | 1 - 2 files changed, 2 deletions(-) diff --git a/go.mod b/go.mod index ac906d6b6bf..5eeb2656499 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.16 require ( github.com/AlekSi/gocov-xml v1.0.0 github.com/BurntSushi/toml v0.3.1 - github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/aws/aws-sdk-go v1.35.3 github.com/axw/gocov v1.0.0 github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 diff --git a/go.sum b/go.sum index 702a4b7866c..50997691e2a 100644 --- a/go.sum +++ b/go.sum @@ -24,7 +24,6 @@ github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQ github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= From 6d2f26d37cafc4b0e327cd39ae91f3935d22854a Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 31 May 2022 16:04:04 +0800 Subject: [PATCH 25/34] update test/client/go.mod Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- tests/client/go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/client/go.sum b/tests/client/go.sum index 3f83cb9a19b..493b1cd39f4 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -24,7 +24,6 @@ github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQ github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= From 3ab7d4f94a4d08abf385f9a18f7eebb1163a3603 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 7 Jun 2022 15:47:50 +0800 Subject: [PATCH 26/34] small cleanup Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 1 - server/storage/kv/etcd_kv.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index 2bb5c9cdc1c..e38d2a4fd4e 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -33,7 +33,6 @@ type KeySpaceGCSafePoint struct { // KeySpaceGCSafePointStorage defines the storage operations on KeySpaces' safe points type KeySpaceGCSafePointStorage interface { // Service safe point interfaces. - // NOTE: field ServiceSafePoint.ExpiredAt will be ignored, use etcd's lease to manage lifetime instead. SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint, ttl int64) error LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) LoadMinServiceSafePoint(spaceID string) (*ServiceSafePoint, error) diff --git a/server/storage/kv/etcd_kv.go b/server/storage/kv/etcd_kv.go index 7efeb61ae1f..abc7d80df40 100644 --- a/server/storage/kv/etcd_kv.go +++ b/server/storage/kv/etcd_kv.go @@ -93,8 +93,8 @@ func (kv *etcdKVBase) LoadRange(key, endKey string, limit int) ([]string, []stri func (kv *etcdKVBase) SaveWithTTL(key, value string, ttlSeconds int64) error { key = path.Join(kv.rootPath, key) - ctx, cancel := context.WithTimeout(kv.client.Ctx(), requestTimeout) start := time.Now() + ctx, cancel := context.WithTimeout(kv.client.Ctx(), requestTimeout) resp, err := etcdutil.EtcdKVPutWithTTL(ctx, kv.client, key, value, ttlSeconds) cancel() From a379eea9920e76b9505e1fa218fd0223dc0351b8 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 8 Jun 2022 11:10:45 +0800 Subject: [PATCH 27/34] avoid modification to general kv interface Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 38 ++++++++++++++++++++++++- server/storage/endpoint/key_path.go | 6 ++++ server/storage/kv/etcd_kv.go | 21 ++++++++------ server/storage/kv/kv.go | 6 ---- server/storage/kv/levedb_kv.go | 11 ------- server/storage/kv/mem_kv.go | 11 ------- 6 files changed, 55 insertions(+), 38 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index e38d2a4fd4e..fb96d3a03cc 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/pingcap/errors" + "github.com/tikv/pd/server/storage/kv" "go.etcd.io/etcd/clientv3" ) @@ -41,6 +42,9 @@ type KeySpaceGCSafePointStorage interface { SaveKeySpaceGCSafePoint(spaceID string, safePoint uint64) error LoadKeySpaceGCSafePoint(spaceID string) (uint64, error) LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([]*KeySpaceGCSafePoint, error) + // Revision interfaces. + TouchKeySpaceRevision(spaceID string) error + LoadKeySpaceRevision(spaceID string) (int64, error) } var _ KeySpaceGCSafePointStorage = (*StorageEndpoint)(nil) @@ -50,12 +54,16 @@ func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafe if ssp.ServiceID == "" { return errors.New("service id of service safepoint cannot be empty") } + etcdEndpoint, err := se.getEtcdBase() + if err != nil { + return err + } key := KeySpaceServiceSafePointPath(spaceID, ssp.ServiceID) value, err := json.Marshal(ssp) if err != nil { return err } - return se.SaveWithTTL(key, string(value), ttl) + return etcdEndpoint.SaveWithTTL(key, string(value), ttl) } // LoadServiceSafePoint reads ServiceSafePoint for the given key-space ID and service name. @@ -159,3 +167,31 @@ func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([] } return safePoints, nil } + +// TouchKeySpaceRevision advances revision of the given key space. +// It's used when new service safe point is saved. +func (se *StorageEndpoint) TouchKeySpaceRevision(spaceID string) error { + path := KeySpacePath(spaceID) + return se.Save(path, "") +} + +// LoadKeySpaceRevision loads the revision of the given key space. +func (se *StorageEndpoint) LoadKeySpaceRevision(spaceID string) (int64, error) { + etcdEndpoint, err := se.getEtcdBase() + if err != nil { + return 0, err + } + keySpacePath := KeySpacePath(spaceID) + _, revision, err := etcdEndpoint.LoadRevision(keySpacePath) + return revision, err +} + +// getEtcdBase retrieves etcd base from storage endpoint. +// It's used by operations that needs etcd endpoint specifically. +func (se *StorageEndpoint) getEtcdBase() (*kv.EtcdKVBase, error) { + etcdBase, ok := interface{}(se.Base).(*kv.EtcdKVBase) + if !ok { + return nil, errors.New("safepoint storage only supports etcd backend") + } + return etcdBase, nil +} diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index db7032b654b..f01891576ac 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -107,6 +107,12 @@ func MinResolvedTSPath() string { return path.Join(clusterPath, minResolvedTS) } +// KeySpacePath returns path to given key space +// Path: /key_space/gc_safepoint/{space_id} +func KeySpacePath(spaceID string) string { + return path.Join(keySpaceSafePointPrefix, spaceID) +} + // KeySpaceServiceSafePointPrefix returns the prefix of given service's service safe point. // Prefix: /key_space/gc_safepoint/{space_id}/service/ func KeySpaceServiceSafePointPrefix(spaceID string) string { diff --git a/server/storage/kv/etcd_kv.go b/server/storage/kv/etcd_kv.go index abc7d80df40..6cc0fee9ba1 100644 --- a/server/storage/kv/etcd_kv.go +++ b/server/storage/kv/etcd_kv.go @@ -32,28 +32,31 @@ import ( const ( requestTimeout = 10 * time.Second slowRequestTime = 1 * time.Second + // RevisionUnavailable is the value of unavailable revision, + // when the kv does not exist. + RevisionUnavailable = -1 ) -type etcdKVBase struct { +type EtcdKVBase struct { client *clientv3.Client rootPath string } // NewEtcdKVBase creates a new etcd kv. -func NewEtcdKVBase(client *clientv3.Client, rootPath string) *etcdKVBase { - return &etcdKVBase{ +func NewEtcdKVBase(client *clientv3.Client, rootPath string) *EtcdKVBase { + return &EtcdKVBase{ client: client, rootPath: rootPath, } } -func (kv *etcdKVBase) Load(key string) (string, error) { +func (kv *EtcdKVBase) Load(key string) (string, error) { value, _, err := kv.LoadRevision(key) return value, err } // LoadRevision gets a value along with revision. -func (kv *etcdKVBase) LoadRevision(key string) (string, int64, error) { +func (kv *EtcdKVBase) LoadRevision(key string) (string, int64, error) { key = path.Join(kv.rootPath, key) resp, err := etcdutil.EtcdKVGet(kv.client, key) @@ -68,7 +71,7 @@ func (kv *etcdKVBase) LoadRevision(key string) (string, int64, error) { return string(resp.Kvs[0].Value), resp.Kvs[0].ModRevision, nil } -func (kv *etcdKVBase) LoadRange(key, endKey string, limit int) ([]string, []string, error) { +func (kv *EtcdKVBase) LoadRange(key, endKey string, limit int) ([]string, []string, error) { // Note: reason to use `strings.Join` instead of `path.Join` is that the latter will // removes suffix '/' of the joined string. // As a result, when we try to scan from "foo/", it ends up scanning from "/pd/foo" @@ -91,7 +94,7 @@ func (kv *etcdKVBase) LoadRange(key, endKey string, limit int) ([]string, []stri return keys, values, nil } -func (kv *etcdKVBase) SaveWithTTL(key, value string, ttlSeconds int64) error { +func (kv *EtcdKVBase) SaveWithTTL(key, value string, ttlSeconds int64) error { key = path.Join(kv.rootPath, key) start := time.Now() ctx, cancel := context.WithTimeout(kv.client.Ctx(), requestTimeout) @@ -119,7 +122,7 @@ func (kv *etcdKVBase) SaveWithTTL(key, value string, ttlSeconds int64) error { return nil } -func (kv *etcdKVBase) Save(key, value string) error { +func (kv *EtcdKVBase) Save(key, value string) error { failpoint.Inject("etcdSaveFailed", func() { failpoint.Return(errors.New("save failed")) }) @@ -137,7 +140,7 @@ func (kv *etcdKVBase) Save(key, value string) error { return nil } -func (kv *etcdKVBase) Remove(key string) error { +func (kv *EtcdKVBase) Remove(key string) error { key = path.Join(kv.rootPath, key) txn := NewSlowLogTxn(kv.client) diff --git a/server/storage/kv/kv.go b/server/storage/kv/kv.go index f7fd609302a..2f1fa06e144 100644 --- a/server/storage/kv/kv.go +++ b/server/storage/kv/kv.go @@ -14,16 +14,10 @@ package kv -// RevisionUnavailable is the value of unavailable revision, -// when the kv does not exist (etcd_kv), or is not supported (mem_kv & leveldb_kv). -const RevisionUnavailable = -1 - // Base is an abstract interface for load/save pd cluster data. type Base interface { Load(key string) (string, error) LoadRange(key, endKey string, limit int) (keys []string, values []string, err error) - LoadRevision(key string) (string, int64, error) Save(key, value string) error - SaveWithTTL(key, value string, ttlSeconds int64) error Remove(key string) error } diff --git a/server/storage/kv/levedb_kv.go b/server/storage/kv/levedb_kv.go index 9f2ed2770ab..7f134709bd1 100644 --- a/server/storage/kv/levedb_kv.go +++ b/server/storage/kv/levedb_kv.go @@ -49,12 +49,6 @@ func (kv *LevelDBKV) Load(key string) (string, error) { return string(v), err } -// LoadRevision gets a value along with revision. The revision is unavailable for `LevelDBKV`. -func (kv *LevelDBKV) LoadRevision(key string) (string, int64, error) { - value, err := kv.Load(key) - return value, RevisionUnavailable, err -} - // LoadRange gets a range of value for a given key range. func (kv *LevelDBKV) LoadRange(startKey, endKey string, limit int) ([]string, []string, error) { iter := kv.NewIterator(&util.Range{Start: []byte(startKey), Limit: []byte(endKey)}, nil) @@ -78,11 +72,6 @@ func (kv *LevelDBKV) Save(key, value string) error { return errors.WithStack(kv.Put([]byte(key), []byte(value), nil)) } -// SaveWithTTL not supported on LevelDBKV -func (kv *LevelDBKV) SaveWithTTL(key, value string, ttlSeconds int64) error { - return errors.New("ttl operation not supported on LevelDBKV") -} - // Remove deletes a key-value pair for a given key. func (kv *LevelDBKV) Remove(key string) error { return errors.WithStack(kv.Delete([]byte(key), nil)) diff --git a/server/storage/kv/mem_kv.go b/server/storage/kv/mem_kv.go index 937a8331277..b74cab84b11 100644 --- a/server/storage/kv/mem_kv.go +++ b/server/storage/kv/mem_kv.go @@ -51,12 +51,6 @@ func (kv *memoryKV) Load(key string) (string, error) { return item.(memoryKVItem).value, nil } -// LoadRevision gets a value along with revision. The revision is unavailable for `memoryKV`. -func (kv *memoryKV) LoadRevision(key string) (string, int64, error) { - value, err := kv.Load(key) - return value, RevisionUnavailable, err -} - func (kv *memoryKV) LoadRange(key, endKey string, limit int) ([]string, []string, error) { failpoint.Inject("withRangeLimit", func(val failpoint.Value) { rangeLimit, ok := val.(int) @@ -86,11 +80,6 @@ func (kv *memoryKV) Save(key, value string) error { return nil } -// SaveWithTTL not supported on memoryKV -func (kv *memoryKV) SaveWithTTL(key, value string, ttlSeconds int64) error { - return errors.New("ttl operation not supported on memoryKV") -} - func (kv *memoryKV) Remove(key string) error { kv.Lock() defer kv.Unlock() From a8790d5b585fe205e4d329eb268556725368ff43 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 8 Jun 2022 15:11:52 +0800 Subject: [PATCH 28/34] storage: handle case where ttl is maxInt Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index fb96d3a03cc..943e0dd4daf 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -63,6 +63,10 @@ func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafe if err != nil { return err } + // A MaxInt64 ttl means safe point never expire. + if ttl == math.MaxInt64 { + return etcdEndpoint.Save(key, string(value)) + } return etcdEndpoint.SaveWithTTL(key, string(value), ttl) } From 5269affbe84d754e6123f91d3cbc72bd2014f4d5 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 8 Jun 2022 13:09:44 +0800 Subject: [PATCH 29/34] add etcdKVBase comments Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/kv/etcd_kv.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/storage/kv/etcd_kv.go b/server/storage/kv/etcd_kv.go index 6cc0fee9ba1..8a5d4010f4b 100644 --- a/server/storage/kv/etcd_kv.go +++ b/server/storage/kv/etcd_kv.go @@ -37,6 +37,7 @@ const ( RevisionUnavailable = -1 ) +// EtcdKVBase is a kv store using etcd. type EtcdKVBase struct { client *clientv3.Client rootPath string @@ -50,6 +51,7 @@ func NewEtcdKVBase(client *clientv3.Client, rootPath string) *EtcdKVBase { } } +// Load gets a value for a given key. func (kv *EtcdKVBase) Load(key string) (string, error) { value, _, err := kv.LoadRevision(key) return value, err @@ -71,6 +73,7 @@ func (kv *EtcdKVBase) LoadRevision(key string) (string, int64, error) { return string(resp.Kvs[0].Value), resp.Kvs[0].ModRevision, nil } +// LoadRange gets a range of value for a given key range. func (kv *EtcdKVBase) LoadRange(key, endKey string, limit int) ([]string, []string, error) { // Note: reason to use `strings.Join` instead of `path.Join` is that the latter will // removes suffix '/' of the joined string. @@ -94,6 +97,7 @@ func (kv *EtcdKVBase) LoadRange(key, endKey string, limit int) ([]string, []stri return keys, values, nil } +// SaveWithTTL stores a key-value pair that expires after ttlSeconds seconds. func (kv *EtcdKVBase) SaveWithTTL(key, value string, ttlSeconds int64) error { key = path.Join(kv.rootPath, key) start := time.Now() @@ -122,6 +126,7 @@ func (kv *EtcdKVBase) SaveWithTTL(key, value string, ttlSeconds int64) error { return nil } +// Save stores a key-value pair. func (kv *EtcdKVBase) Save(key, value string) error { failpoint.Inject("etcdSaveFailed", func() { failpoint.Return(errors.New("save failed")) @@ -140,6 +145,7 @@ func (kv *EtcdKVBase) Save(key, value string) error { return nil } +// Remove deletes a key-value pair for a given key. func (kv *EtcdKVBase) Remove(key string) error { key = path.Join(kv.rootPath, key) From 7c05203d485eb554be10bbb227da8ae2eba20436 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 8 Jun 2022 17:50:50 +0800 Subject: [PATCH 30/34] storage: Revision should be int64 Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/kv/etcd_kv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/storage/kv/etcd_kv.go b/server/storage/kv/etcd_kv.go index 8a5d4010f4b..fa5072be53c 100644 --- a/server/storage/kv/etcd_kv.go +++ b/server/storage/kv/etcd_kv.go @@ -34,7 +34,7 @@ const ( slowRequestTime = 1 * time.Second // RevisionUnavailable is the value of unavailable revision, // when the kv does not exist. - RevisionUnavailable = -1 + RevisionUnavailable int64 = -1 ) // EtcdKVBase is a kv store using etcd. From 77d6f26039021542292e5e98629b5f53c10fe945 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 8 Jun 2022 18:11:16 +0800 Subject: [PATCH 31/34] storage: add revision tests Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/storage_gc_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index 24ed77be9ec..0743d38170d 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -26,6 +26,7 @@ import ( . "github.com/pingcap/check" "github.com/tikv/pd/pkg/tempurl" "github.com/tikv/pd/server/storage/endpoint" + "github.com/tikv/pd/server/storage/kv" "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/embed" ) @@ -223,6 +224,21 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { c.Assert(loadedSafePoints[i].SafePoint, Equals, uint64(0)) } } +func (s *testStorageGCSuite) TestRevision(c *C) { + storage := s.storage + // Touching KeySpace2 should not change revision of KeySpace1 + c.Assert(storage.TouchKeySpaceRevision("KeySpace1"), IsNil) + oldRevision, err := storage.LoadKeySpaceRevision("KeySpace1") + c.Assert(err, IsNil) + c.Assert(storage.TouchKeySpaceRevision("KeySpace2"), IsNil) + newRevision, err := storage.LoadKeySpaceRevision("KeySpace1") + c.Assert(oldRevision, Equals, newRevision) + + // Touching the same key space should change revision + c.Assert(storage.TouchKeySpaceRevision("KeySpace1"), IsNil) + newRevision, err = storage.LoadKeySpaceRevision("KeySpace1") + c.Assert(oldRevision, Not(Equals), newRevision) +} func (s *testStorageGCSuite) TestLoadEmpty(c *C) { storage := s.storage @@ -241,6 +257,11 @@ func (s *testStorageGCSuite) TestLoadEmpty(c *C) { safePoints, err := storage.LoadAllKeySpaceGCSafePoints(true) c.Assert(err, IsNil) c.Assert(safePoints, HasLen, 0) + + // Loading untouched key spaces should return unavailable revision + revision, err := storage.LoadKeySpaceRevision("testKeySpace") + c.Assert(err, IsNil) + c.Assert(revision, Equals, kv.RevisionUnavailable) } func newTestSingleConfig() *embed.Config { From bda31f45fc8bd19b8bcf86196903501b50d22e0c Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 8 Jun 2022 18:16:11 +0800 Subject: [PATCH 32/34] storage: lint Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/storage_gc_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index 0743d38170d..f5b64e5c5d7 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -232,11 +232,13 @@ func (s *testStorageGCSuite) TestRevision(c *C) { c.Assert(err, IsNil) c.Assert(storage.TouchKeySpaceRevision("KeySpace2"), IsNil) newRevision, err := storage.LoadKeySpaceRevision("KeySpace1") + c.Assert(err, IsNil) c.Assert(oldRevision, Equals, newRevision) // Touching the same key space should change revision c.Assert(storage.TouchKeySpaceRevision("KeySpace1"), IsNil) newRevision, err = storage.LoadKeySpaceRevision("KeySpace1") + c.Assert(err, IsNil) c.Assert(oldRevision, Not(Equals), newRevision) } From 9ff8090938372e9f2ebb521f13a3b5c79dc86e06 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 14 Jun 2022 15:32:18 +0800 Subject: [PATCH 33/34] storage: change spaceID to uint32 Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/endpoint/gc_key_space.go | 55 +++++++++++-------- server/storage/endpoint/key_path.go | 16 +++--- server/storage/storage_gc_test.go | 72 ++++++++++++++----------- 3 files changed, 83 insertions(+), 60 deletions(-) diff --git a/server/storage/endpoint/gc_key_space.go b/server/storage/endpoint/gc_key_space.go index 943e0dd4daf..fac27728972 100644 --- a/server/storage/endpoint/gc_key_space.go +++ b/server/storage/endpoint/gc_key_space.go @@ -25,32 +25,37 @@ import ( "go.etcd.io/etcd/clientv3" ) +const ( + spaceIDBase = 16 + safePointBase = 16 +) + // KeySpaceGCSafePoint is gcWorker's safepoint for specific key-space type KeySpaceGCSafePoint struct { - SpaceID string `json:"space_id"` + SpaceID uint32 `json:"space_id"` SafePoint uint64 `json:"safe_point,omitempty"` } // KeySpaceGCSafePointStorage defines the storage operations on KeySpaces' safe points type KeySpaceGCSafePointStorage interface { // Service safe point interfaces. - SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint, ttl int64) error - LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) - LoadMinServiceSafePoint(spaceID string) (*ServiceSafePoint, error) - RemoveServiceSafePoint(spaceID, serviceID string) error + SaveServiceSafePoint(spaceID uint32, ssp *ServiceSafePoint, ttl int64) error + LoadServiceSafePoint(spaceID uint32, serviceID string) (*ServiceSafePoint, error) + LoadMinServiceSafePoint(spaceID uint32) (*ServiceSafePoint, error) + RemoveServiceSafePoint(spaceID uint32, serviceID string) error // GC safe point interfaces. - SaveKeySpaceGCSafePoint(spaceID string, safePoint uint64) error - LoadKeySpaceGCSafePoint(spaceID string) (uint64, error) + SaveKeySpaceGCSafePoint(spaceID uint32, safePoint uint64) error + LoadKeySpaceGCSafePoint(spaceID uint32) (uint64, error) LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([]*KeySpaceGCSafePoint, error) // Revision interfaces. - TouchKeySpaceRevision(spaceID string) error - LoadKeySpaceRevision(spaceID string) (int64, error) + TouchKeySpaceRevision(spaceID uint32) error + LoadKeySpaceRevision(spaceID uint32) (int64, error) } var _ KeySpaceGCSafePointStorage = (*StorageEndpoint)(nil) // SaveServiceSafePoint saves service safe point under given key-space. -func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafePoint, ttl int64) error { +func (se *StorageEndpoint) SaveServiceSafePoint(spaceID uint32, ssp *ServiceSafePoint, ttl int64) error { if ssp.ServiceID == "" { return errors.New("service id of service safepoint cannot be empty") } @@ -72,7 +77,7 @@ func (se *StorageEndpoint) SaveServiceSafePoint(spaceID string, ssp *ServiceSafe // LoadServiceSafePoint reads ServiceSafePoint for the given key-space ID and service name. // Return nil if no safepoint exist for given service. -func (se *StorageEndpoint) LoadServiceSafePoint(spaceID, serviceID string) (*ServiceSafePoint, error) { +func (se *StorageEndpoint) LoadServiceSafePoint(spaceID uint32, serviceID string) (*ServiceSafePoint, error) { key := KeySpaceServiceSafePointPath(spaceID, serviceID) value, err := se.Load(key) if err != nil || value == "" { @@ -88,7 +93,7 @@ func (se *StorageEndpoint) LoadServiceSafePoint(spaceID, serviceID string) (*Ser // LoadMinServiceSafePoint returns the minimum safepoint for the given key-space. // Note that gc worker safe point are store separately. // If no service safe point exist for the given key-space or all the service safe points just expired, return nil. -func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string) (*ServiceSafePoint, error) { +func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID uint32) (*ServiceSafePoint, error) { prefix := KeySpaceServiceSafePointPrefix(spaceID) prefixEnd := clientv3.GetPrefixRangeEnd(prefix) _, values, err := se.LoadRange(prefix, prefixEnd, 0) @@ -115,25 +120,25 @@ func (se *StorageEndpoint) LoadMinServiceSafePoint(spaceID string) (*ServiceSafe } // RemoveServiceSafePoint removes target ServiceSafePoint -func (se *StorageEndpoint) RemoveServiceSafePoint(spaceID, serviceID string) error { +func (se *StorageEndpoint) RemoveServiceSafePoint(spaceID uint32, serviceID string) error { key := KeySpaceServiceSafePointPath(spaceID, serviceID) return se.Remove(key) } // SaveKeySpaceGCSafePoint saves GCSafePoint to the given key-space. -func (se *StorageEndpoint) SaveKeySpaceGCSafePoint(spaceID string, safePoint uint64) error { - value := strconv.FormatUint(safePoint, 16) +func (se *StorageEndpoint) SaveKeySpaceGCSafePoint(spaceID uint32, safePoint uint64) error { + value := strconv.FormatUint(safePoint, safePointBase) return se.Save(KeySpaceGCSafePointPath(spaceID), value) } // LoadKeySpaceGCSafePoint reads GCSafePoint for the given key-space. // Returns 0 if target safepoint not exist. -func (se *StorageEndpoint) LoadKeySpaceGCSafePoint(spaceID string) (uint64, error) { +func (se *StorageEndpoint) LoadKeySpaceGCSafePoint(spaceID uint32) (uint64, error) { value, err := se.Load(KeySpaceGCSafePointPath(spaceID)) if err != nil || value == "" { return 0, err } - safePoint, err := strconv.ParseUint(value, 16, 64) + safePoint, err := strconv.ParseUint(value, safePointBase, 64) if err != nil { return 0, err } @@ -157,11 +162,15 @@ func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([] continue } safePoint := &KeySpaceGCSafePoint{} - spaceID := strings.TrimPrefix(keys[i], prefix) - spaceID = strings.TrimSuffix(spaceID, suffix) - safePoint.SpaceID = spaceID + spaceIDStr := strings.TrimPrefix(keys[i], prefix) + spaceIDStr = strings.TrimSuffix(spaceIDStr, suffix) + spaceID, err := strconv.ParseUint(spaceIDStr, spaceIDBase, 32) + if err != nil { + return nil, err + } + safePoint.SpaceID = uint32(spaceID) if withGCSafePoint { - value, err := strconv.ParseUint(values[i], 16, 64) + value, err := strconv.ParseUint(values[i], safePointBase, 64) if err != nil { return nil, err } @@ -174,13 +183,13 @@ func (se *StorageEndpoint) LoadAllKeySpaceGCSafePoints(withGCSafePoint bool) ([] // TouchKeySpaceRevision advances revision of the given key space. // It's used when new service safe point is saved. -func (se *StorageEndpoint) TouchKeySpaceRevision(spaceID string) error { +func (se *StorageEndpoint) TouchKeySpaceRevision(spaceID uint32) error { path := KeySpacePath(spaceID) return se.Save(path, "") } // LoadKeySpaceRevision loads the revision of the given key space. -func (se *StorageEndpoint) LoadKeySpaceRevision(spaceID string) (int64, error) { +func (se *StorageEndpoint) LoadKeySpaceRevision(spaceID uint32) (int64, error) { etcdEndpoint, err := se.getEtcdBase() if err != nil { return 0, err diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index f01891576ac..0676e437709 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -17,6 +17,7 @@ package endpoint import ( "fmt" "path" + "strconv" ) const ( @@ -109,25 +110,26 @@ func MinResolvedTSPath() string { // KeySpacePath returns path to given key space // Path: /key_space/gc_safepoint/{space_id} -func KeySpacePath(spaceID string) string { - return path.Join(keySpaceSafePointPrefix, spaceID) +func KeySpacePath(spaceID uint32) string { + spaceIDStr := strconv.FormatUint(uint64(spaceID), spaceIDBase) + return path.Join(keySpaceSafePointPrefix, spaceIDStr) } // KeySpaceServiceSafePointPrefix returns the prefix of given service's service safe point. // Prefix: /key_space/gc_safepoint/{space_id}/service/ -func KeySpaceServiceSafePointPrefix(spaceID string) string { - return path.Join(keySpaceSafePointPrefix, spaceID, "service") + "/" +func KeySpaceServiceSafePointPrefix(spaceID uint32) string { + return path.Join(KeySpacePath(spaceID), "service") + "/" } // KeySpaceGCSafePointPath returns the gc safe point's path of the given key-space. // Path: /key_space/gc_safepoint/{space_id}/gc -func KeySpaceGCSafePointPath(spaceID string) string { - return path.Join(keySpaceSafePointPrefix, spaceID, keySpaceGCSafePointSuffix) +func KeySpaceGCSafePointPath(spaceID uint32) string { + return path.Join(KeySpacePath(spaceID), keySpaceGCSafePointSuffix) } // KeySpaceServiceSafePointPath returns the path of given service's service safe point. // Path: /key_space/gc_safepoint/{space_id}/service/{service_id} -func KeySpaceServiceSafePointPath(spaceID, serviceID string) string { +func KeySpaceServiceSafePointPath(spaceID uint32, serviceID string) string { return path.Join(KeySpaceServiceSafePointPrefix(spaceID), serviceID) } diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index f5b64e5c5d7..a0e1763afd4 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -20,6 +20,7 @@ import ( "net/url" "os" "path" + "sort" "strconv" "time" @@ -59,13 +60,13 @@ func (s *testStorageGCSuite) TearDownTest(c *C) { c.Assert(cleanConfig(s.cfg), IsNil) } -func testGCSafePoints() ([]string, []uint64) { - spaceIDs := []string{ - "keySpace1", - "keySpace2", - "keySpace3", - "keySpace4", - "keySpace5", +func testGCSafePoints() ([]uint32, []uint64) { + spaceIDs := []uint32{ + 100, + 200, + 300, + 400, + 500, } safePoints := []uint64{ 0, @@ -77,17 +78,17 @@ func testGCSafePoints() ([]string, []uint64) { return spaceIDs, safePoints } -func testServiceSafePoints() ([]string, []*endpoint.ServiceSafePoint, []int64) { - spaceIDs := []string{ - "keySpace1", - "keySpace1", - "keySpace1", - "keySpace2", - "keySpace2", - "keySpace2", - "keySpace3", - "keySpace3", - "keySpace3", +func testServiceSafePoints() ([]uint32, []*endpoint.ServiceSafePoint, []int64) { + spaceIDs := []uint32{ + 100, + 100, + 100, + 200, + 200, + 200, + 300, + 300, + 300, } serviceSafePoints := []*endpoint.ServiceSafePoint{ {ServiceID: "service1", SafePoint: 1}, @@ -127,7 +128,7 @@ func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { {ServiceID: "0", SafePoint: 100}, {ServiceID: "1", SafePoint: 200}, } - testKeySpace := "test" + testKeySpace := uint32(100) for i := range serviceSafePoints { c.Assert(storage.SaveServiceSafePoint(testKeySpace, serviceSafePoints[i], testTTLs[i]), IsNil) } @@ -197,6 +198,9 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { } loadedSafePoints, err := storage.LoadAllKeySpaceGCSafePoints(true) c.Assert(err, IsNil) + sort.Slice(loadedSafePoints, func(a, b int) bool { + return loadedSafePoints[a].SpaceID < loadedSafePoints[b].SpaceID + }) for i := range loadedSafePoints { c.Assert(loadedSafePoints[i].SpaceID, Equals, testSpaceIDs[i]) c.Assert(loadedSafePoints[i].SafePoint, Equals, testSafePoints[i]) @@ -211,6 +215,9 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { // verify that service safe points do not interfere with gc safe points. loadedSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(true) c.Assert(err, IsNil) + sort.Slice(loadedSafePoints, func(a, b int) bool { + return loadedSafePoints[a].SpaceID < loadedSafePoints[b].SpaceID + }) for i := range loadedSafePoints { c.Assert(loadedSafePoints[i].SpaceID, Equals, testSpaceIDs[i]) c.Assert(loadedSafePoints[i].SafePoint, Equals, testSafePoints[i]) @@ -219,6 +226,9 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { // verify that when withGCSafePoint set to false, returned safePoints is 0 loadedSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(false) c.Assert(err, IsNil) + sort.Slice(loadedSafePoints, func(a, b int) bool { + return loadedSafePoints[a].SpaceID < loadedSafePoints[b].SpaceID + }) for i := range loadedSafePoints { c.Assert(loadedSafePoints[i].SpaceID, Equals, testSpaceIDs[i]) c.Assert(loadedSafePoints[i].SafePoint, Equals, uint64(0)) @@ -226,32 +236,34 @@ func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { } func (s *testStorageGCSuite) TestRevision(c *C) { storage := s.storage - // Touching KeySpace2 should not change revision of KeySpace1 - c.Assert(storage.TouchKeySpaceRevision("KeySpace1"), IsNil) - oldRevision, err := storage.LoadKeySpaceRevision("KeySpace1") + keySpace1 := uint32(100) + keySpace2 := uint32(200) + // Touching key space 200 should not change revision of key space 100 + c.Assert(storage.TouchKeySpaceRevision(keySpace1), IsNil) + oldRevision, err := storage.LoadKeySpaceRevision(keySpace1) c.Assert(err, IsNil) - c.Assert(storage.TouchKeySpaceRevision("KeySpace2"), IsNil) - newRevision, err := storage.LoadKeySpaceRevision("KeySpace1") + c.Assert(storage.TouchKeySpaceRevision(keySpace2), IsNil) + newRevision, err := storage.LoadKeySpaceRevision(keySpace1) c.Assert(err, IsNil) c.Assert(oldRevision, Equals, newRevision) // Touching the same key space should change revision - c.Assert(storage.TouchKeySpaceRevision("KeySpace1"), IsNil) - newRevision, err = storage.LoadKeySpaceRevision("KeySpace1") + c.Assert(storage.TouchKeySpaceRevision(keySpace1), IsNil) + newRevision, err = storage.LoadKeySpaceRevision(keySpace1) c.Assert(err, IsNil) c.Assert(oldRevision, Not(Equals), newRevision) } func (s *testStorageGCSuite) TestLoadEmpty(c *C) { storage := s.storage - + testKeySpace := uint32(100) // loading non-existing GC safepoint should return 0 - gcSafePoint, err := storage.LoadKeySpaceGCSafePoint("testKeySpace") + gcSafePoint, err := storage.LoadKeySpaceGCSafePoint(testKeySpace) c.Assert(err, IsNil) c.Assert(gcSafePoint, Equals, uint64(0)) // loading non-existing service safepoint should return nil - serviceSafePoint, err := storage.LoadServiceSafePoint("testKeySpace", "testService") + serviceSafePoint, err := storage.LoadServiceSafePoint(testKeySpace, "testService") c.Assert(err, IsNil) c.Assert(serviceSafePoint, IsNil) @@ -261,7 +273,7 @@ func (s *testStorageGCSuite) TestLoadEmpty(c *C) { c.Assert(safePoints, HasLen, 0) // Loading untouched key spaces should return unavailable revision - revision, err := storage.LoadKeySpaceRevision("testKeySpace") + revision, err := storage.LoadKeySpaceRevision(testKeySpace) c.Assert(err, IsNil) c.Assert(revision, Equals, kv.RevisionUnavailable) } From ffe8a183720ea8c876b083ff64a07d47d52c6240 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Tue, 21 Jun 2022 23:56:05 +0800 Subject: [PATCH 34/34] storage: migrate tests to testify Signed-off-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- server/storage/storage_gc_test.go | 254 +++++++++++++++--------------- 1 file changed, 123 insertions(+), 131 deletions(-) diff --git a/server/storage/storage_gc_test.go b/server/storage/storage_gc_test.go index a0e1763afd4..21b4d19c116 100644 --- a/server/storage/storage_gc_test.go +++ b/server/storage/storage_gc_test.go @@ -22,9 +22,10 @@ import ( "path" "sort" "strconv" + "testing" "time" - . "github.com/pingcap/check" + "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/tempurl" "github.com/tikv/pd/server/storage/endpoint" "github.com/tikv/pd/server/storage/kv" @@ -32,97 +33,53 @@ import ( "go.etcd.io/etcd/embed" ) -var _ = Suite(&testStorageGCSuite{}) +func TestStorageGCTestSuite(t *testing.T) { + suite.Run(t, new(StorageGCTestSuite)) +} -type testStorageGCSuite struct { +type StorageGCTestSuite struct { + suite.Suite cfg *embed.Config etcd *embed.Etcd storage Storage } -func (s *testStorageGCSuite) SetUpTest(c *C) { - s.cfg = newTestSingleConfig() - s.etcd = mustNewEmbedEtcd(c, s.cfg) - - ep := s.cfg.LCUrls[0].String() +func (suite *StorageGCTestSuite) SetupTest() { + var err error + suite.cfg = newTestSingleConfig() + suite.etcd, err = embed.StartEtcd(suite.cfg) + suite.Require().Nil(err) + ep := suite.cfg.LCUrls[0].String() client, err := clientv3.New(clientv3.Config{ Endpoints: []string{ep}, }) - c.Assert(err, IsNil) + suite.Require().Nil(err) rootPath := path.Join("/pd", strconv.FormatUint(100, 10)) - s.storage = NewStorageWithEtcdBackend(client, rootPath) -} - -func (s *testStorageGCSuite) TearDownTest(c *C) { - if s.etcd != nil { - s.etcd.Close() - } - c.Assert(cleanConfig(s.cfg), IsNil) -} - -func testGCSafePoints() ([]uint32, []uint64) { - spaceIDs := []uint32{ - 100, - 200, - 300, - 400, - 500, - } - safePoints := []uint64{ - 0, - 1, - 4396, - 23333333333, - math.MaxUint64, - } - return spaceIDs, safePoints + suite.storage = NewStorageWithEtcdBackend(client, rootPath) } -func testServiceSafePoints() ([]uint32, []*endpoint.ServiceSafePoint, []int64) { - spaceIDs := []uint32{ - 100, - 100, - 100, - 200, - 200, - 200, - 300, - 300, - 300, - } - serviceSafePoints := []*endpoint.ServiceSafePoint{ - {ServiceID: "service1", SafePoint: 1}, - {ServiceID: "service2", SafePoint: 2}, - {ServiceID: "service3", SafePoint: 3}, - {ServiceID: "service1", SafePoint: 1}, - {ServiceID: "service2", SafePoint: 2}, - {ServiceID: "service3", SafePoint: 3}, - {ServiceID: "service1", SafePoint: 1}, - {ServiceID: "service2", SafePoint: 2}, - {ServiceID: "service3", SafePoint: 3}, - } - testTTls := make([]int64, 9) - for i := range testTTls { - testTTls[i] = 10 +func (suite *StorageGCTestSuite) TearDownTest() { + if suite.etcd != nil { + suite.etcd.Close() } - return spaceIDs, serviceSafePoints, testTTls + suite.Require().NoError(os.RemoveAll(suite.cfg.Dir)) } -func (s *testStorageGCSuite) TestSaveLoadServiceSafePoint(c *C) { - storage := s.storage +func (suite *StorageGCTestSuite) TestSaveLoadServiceSafePoint() { + storage := suite.storage testSpaceID, testSafePoints, testTTLs := testServiceSafePoints() for i := range testSpaceID { - c.Assert(storage.SaveServiceSafePoint(testSpaceID[i], testSafePoints[i], testTTLs[i]), IsNil) + suite.NoError(storage.SaveServiceSafePoint(testSpaceID[i], testSafePoints[i], testTTLs[i])) } for i := range testSpaceID { loadedSafePoint, err := storage.LoadServiceSafePoint(testSpaceID[i], testSafePoints[i].ServiceID) - c.Assert(err, IsNil) - c.Assert(loadedSafePoint, DeepEquals, testSafePoints[i]) + suite.Nil(err) + suite.Equal(testSafePoints[i], loadedSafePoint) } } -func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { - storage := s.storage +func (suite *StorageGCTestSuite) TestLoadMinServiceSafePoint() { + storage := suite.storage testTTLs := []int64{2, 6} serviceSafePoints := []*endpoint.ServiceSafePoint{ {ServiceID: "0", SafePoint: 100}, @@ -130,152 +87,150 @@ func (s *testStorageGCSuite) TestLoadMinServiceSafePoint(c *C) { } testKeySpace := uint32(100) for i := range serviceSafePoints { - c.Assert(storage.SaveServiceSafePoint(testKeySpace, serviceSafePoints[i], testTTLs[i]), IsNil) + suite.NoError(storage.SaveServiceSafePoint(testKeySpace, serviceSafePoints[i], testTTLs[i])) } minSafePoint, err := storage.LoadMinServiceSafePoint(testKeySpace) - c.Assert(err, IsNil) - c.Assert(minSafePoint, DeepEquals, serviceSafePoints[0]) + suite.Nil(err) + suite.Equal(serviceSafePoints[0], minSafePoint) time.Sleep(4 * time.Second) // the safePoint with ServiceID 0 should be removed due to expiration // now min should be safePoint with ServiceID 1 minSafePoint2, err := storage.LoadMinServiceSafePoint(testKeySpace) - c.Assert(err, IsNil) - c.Assert(minSafePoint2, DeepEquals, serviceSafePoints[1]) + suite.Nil(err) + suite.Equal(serviceSafePoints[1], minSafePoint2) // verify that service safe point with ServiceID 0 has been removed ssp, err := storage.LoadServiceSafePoint(testKeySpace, "0") - c.Assert(err, IsNil) - c.Assert(ssp, IsNil) + suite.Nil(err) + suite.Nil(ssp) time.Sleep(4 * time.Second) // all remaining service safePoints should be removed due to expiration ssp, err = storage.LoadMinServiceSafePoint(testKeySpace) - c.Assert(err, IsNil) - c.Assert(ssp, IsNil) + suite.Nil(err) + suite.Nil(ssp) } -func (s *testStorageGCSuite) TestRemoveServiceSafePoint(c *C) { - storage := s.storage +func (suite *StorageGCTestSuite) TestRemoveServiceSafePoint() { + storage := suite.storage testSpaceID, testSafePoints, testTTLs := testServiceSafePoints() // save service safe points for i := range testSpaceID { - c.Assert(storage.SaveServiceSafePoint(testSpaceID[i], testSafePoints[i], testTTLs[i]), IsNil) + suite.NoError(storage.SaveServiceSafePoint(testSpaceID[i], testSafePoints[i], testTTLs[i])) } // remove saved service safe points for i := range testSpaceID { - c.Assert(storage.RemoveServiceSafePoint(testSpaceID[i], testSafePoints[i].ServiceID), IsNil) + suite.NoError(storage.RemoveServiceSafePoint(testSpaceID[i], testSafePoints[i].ServiceID)) } // check that service safe points are empty for i := range testSpaceID { loadedSafePoint, err := storage.LoadServiceSafePoint(testSpaceID[i], testSafePoints[i].ServiceID) - c.Assert(err, IsNil) - c.Assert(loadedSafePoint, IsNil) + suite.Nil(err) + suite.Nil(loadedSafePoint) } } -func (s *testStorageGCSuite) TestSaveLoadGCSafePoint(c *C) { - storage := s.storage +func (suite *StorageGCTestSuite) TestSaveLoadGCSafePoint() { + storage := suite.storage testSpaceIDs, testSafePoints := testGCSafePoints() for i := range testSpaceIDs { testSpaceID := testSpaceIDs[i] testSafePoint := testSafePoints[i] - err := storage.SaveKeySpaceGCSafePoint(testSpaceID, testSafePoint) - c.Assert(err, IsNil) + suite.NoError(storage.SaveKeySpaceGCSafePoint(testSpaceID, testSafePoint)) loaded, err := storage.LoadKeySpaceGCSafePoint(testSpaceID) - c.Assert(err, IsNil) - c.Assert(loaded, Equals, testSafePoint) + suite.Nil(err) + suite.Equal(testSafePoint, loaded) } } -func (s *testStorageGCSuite) TestLoadAllKeySpaceGCSafePoints(c *C) { - storage := s.storage +func (suite *StorageGCTestSuite) TestLoadAllKeySpaceGCSafePoints() { + storage := suite.storage testSpaceIDs, testSafePoints := testGCSafePoints() for i := range testSpaceIDs { - err := storage.SaveKeySpaceGCSafePoint(testSpaceIDs[i], testSafePoints[i]) - c.Assert(err, IsNil) + suite.NoError(storage.SaveKeySpaceGCSafePoint(testSpaceIDs[i], testSafePoints[i])) } loadedSafePoints, err := storage.LoadAllKeySpaceGCSafePoints(true) - c.Assert(err, IsNil) + suite.Nil(err) sort.Slice(loadedSafePoints, func(a, b int) bool { return loadedSafePoints[a].SpaceID < loadedSafePoints[b].SpaceID }) for i := range loadedSafePoints { - c.Assert(loadedSafePoints[i].SpaceID, Equals, testSpaceIDs[i]) - c.Assert(loadedSafePoints[i].SafePoint, Equals, testSafePoints[i]) + suite.Equal(testSpaceIDs[i], loadedSafePoints[i].SpaceID) + suite.Equal(testSafePoints[i], loadedSafePoints[i].SafePoint) } // saving some service safe points. spaceIDs, safePoints, TTLs := testServiceSafePoints() for i := range spaceIDs { - c.Assert(storage.SaveServiceSafePoint(spaceIDs[i], safePoints[i], TTLs[i]), IsNil) + suite.NoError(storage.SaveServiceSafePoint(spaceIDs[i], safePoints[i], TTLs[i])) } // verify that service safe points do not interfere with gc safe points. loadedSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(true) - c.Assert(err, IsNil) + suite.Nil(err) sort.Slice(loadedSafePoints, func(a, b int) bool { return loadedSafePoints[a].SpaceID < loadedSafePoints[b].SpaceID }) for i := range loadedSafePoints { - c.Assert(loadedSafePoints[i].SpaceID, Equals, testSpaceIDs[i]) - c.Assert(loadedSafePoints[i].SafePoint, Equals, testSafePoints[i]) + suite.Equal(testSpaceIDs[i], loadedSafePoints[i].SpaceID) + suite.Equal(testSafePoints[i], loadedSafePoints[i].SafePoint) } // verify that when withGCSafePoint set to false, returned safePoints is 0 loadedSafePoints, err = storage.LoadAllKeySpaceGCSafePoints(false) - c.Assert(err, IsNil) + suite.Nil(err) sort.Slice(loadedSafePoints, func(a, b int) bool { return loadedSafePoints[a].SpaceID < loadedSafePoints[b].SpaceID }) for i := range loadedSafePoints { - c.Assert(loadedSafePoints[i].SpaceID, Equals, testSpaceIDs[i]) - c.Assert(loadedSafePoints[i].SafePoint, Equals, uint64(0)) + suite.Equal(testSpaceIDs[i], loadedSafePoints[i].SpaceID) + suite.Equal(uint64(0), loadedSafePoints[i].SafePoint) } } -func (s *testStorageGCSuite) TestRevision(c *C) { - storage := s.storage +func (suite *StorageGCTestSuite) TestRevision() { + storage := suite.storage keySpace1 := uint32(100) keySpace2 := uint32(200) // Touching key space 200 should not change revision of key space 100 - c.Assert(storage.TouchKeySpaceRevision(keySpace1), IsNil) + suite.NoError(storage.TouchKeySpaceRevision(keySpace1)) oldRevision, err := storage.LoadKeySpaceRevision(keySpace1) - c.Assert(err, IsNil) - c.Assert(storage.TouchKeySpaceRevision(keySpace2), IsNil) + suite.Nil(err) + suite.NoError(storage.TouchKeySpaceRevision(keySpace2)) newRevision, err := storage.LoadKeySpaceRevision(keySpace1) - c.Assert(err, IsNil) - c.Assert(oldRevision, Equals, newRevision) + suite.Nil(err) + suite.Equal(oldRevision, newRevision) // Touching the same key space should change revision - c.Assert(storage.TouchKeySpaceRevision(keySpace1), IsNil) + suite.NoError(storage.TouchKeySpaceRevision(keySpace1)) newRevision, err = storage.LoadKeySpaceRevision(keySpace1) - c.Assert(err, IsNil) - c.Assert(oldRevision, Not(Equals), newRevision) + suite.Nil(err) + suite.NotEqual(oldRevision, newRevision) } -func (s *testStorageGCSuite) TestLoadEmpty(c *C) { - storage := s.storage +func (suite *StorageGCTestSuite) TestLoadEmpty() { + storage := suite.storage testKeySpace := uint32(100) // loading non-existing GC safepoint should return 0 gcSafePoint, err := storage.LoadKeySpaceGCSafePoint(testKeySpace) - c.Assert(err, IsNil) - c.Assert(gcSafePoint, Equals, uint64(0)) + suite.Nil(err) + suite.Equal(uint64(0), gcSafePoint) // loading non-existing service safepoint should return nil serviceSafePoint, err := storage.LoadServiceSafePoint(testKeySpace, "testService") - c.Assert(err, IsNil) - c.Assert(serviceSafePoint, IsNil) + suite.Nil(err) + suite.Nil(serviceSafePoint) // loading empty key spaces should return empty slices safePoints, err := storage.LoadAllKeySpaceGCSafePoints(true) - c.Assert(err, IsNil) - c.Assert(safePoints, HasLen, 0) + suite.Nil(err) + suite.Empty(safePoints) // Loading untouched key spaces should return unavailable revision revision, err := storage.LoadKeySpaceRevision(testKeySpace) - c.Assert(err, IsNil) - c.Assert(revision, Equals, kv.RevisionUnavailable) + suite.Nil(err) + suite.Equal(kv.RevisionUnavailable, revision) } func newTestSingleConfig() *embed.Config { @@ -299,13 +254,50 @@ func newTestSingleConfig() *embed.Config { return cfg } -func cleanConfig(cfg *embed.Config) error { - // Clean data directory - return os.RemoveAll(cfg.Dir) +func testGCSafePoints() ([]uint32, []uint64) { + spaceIDs := []uint32{ + 100, + 200, + 300, + 400, + 500, + } + safePoints := []uint64{ + 0, + 1, + 4396, + 23333333333, + math.MaxUint64, + } + return spaceIDs, safePoints } -func mustNewEmbedEtcd(c *C, cfg *embed.Config) *embed.Etcd { - etcd, err := embed.StartEtcd(cfg) - c.Assert(err, IsNil) - return etcd +func testServiceSafePoints() ([]uint32, []*endpoint.ServiceSafePoint, []int64) { + spaceIDs := []uint32{ + 100, + 100, + 100, + 200, + 200, + 200, + 300, + 300, + 300, + } + serviceSafePoints := []*endpoint.ServiceSafePoint{ + {ServiceID: "service1", SafePoint: 1}, + {ServiceID: "service2", SafePoint: 2}, + {ServiceID: "service3", SafePoint: 3}, + {ServiceID: "service1", SafePoint: 1}, + {ServiceID: "service2", SafePoint: 2}, + {ServiceID: "service3", SafePoint: 3}, + {ServiceID: "service1", SafePoint: 1}, + {ServiceID: "service2", SafePoint: 2}, + {ServiceID: "service3", SafePoint: 3}, + } + testTTls := make([]int64, 9) + for i := range testTTls { + testTTls[i] = 10 + } + return spaceIDs, serviceSafePoints, testTTls }