From e8102b4905d15826443a6b563e43c8786906d5cc Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Sun, 24 Apr 2022 16:45:26 +0800 Subject: [PATCH 01/56] add limit config Signed-off-by: Cabinfever_B --- pkg/ratelimit/limiter.go | 34 ++++++++++-- pkg/ratelimit/limiter_test.go | 19 ++++--- pkg/ratelimit/option.go | 99 ++++++++++++++++++++++++++++++----- 3 files changed, 126 insertions(+), 26 deletions(-) diff --git a/pkg/ratelimit/limiter.go b/pkg/ratelimit/limiter.go index 43f01cea41b..ccc93037ea0 100644 --- a/pkg/ratelimit/limiter.go +++ b/pkg/ratelimit/limiter.go @@ -20,8 +20,27 @@ import ( "golang.org/x/time/rate" ) +// LimiterConfig is the config of Limiter +type LimiterConfig map[string]DimensionConfig + +// NewLimiterConfig returns a new LimiterConfig +func NewLimiterConfig() LimiterConfig { + return make(map[string]DimensionConfig) +} + +// DimensionConfig is the limit dimension config of one label +type DimensionConfig struct { + // qps conifg + QPS float64 + QPSBrust int + // concurrency config + ConcurrencyLimit uint64 +} + // Limiter is a controller for the request rate. type Limiter struct { + configMux sync.Mutex + labelConfig LimiterConfig qpsLimiter sync.Map concurrencyLimiter sync.Map // the label which is in labelAllowList won't be limited @@ -30,7 +49,10 @@ type Limiter struct { // NewLimiter returns a global limiter which can be updated in the later. func NewLimiter() *Limiter { - return &Limiter{labelAllowList: make(map[string]struct{})} + return &Limiter{ + labelAllowList: make(map[string]struct{}), + labelConfig: NewLimiterConfig(), + } } // Allow is used to check whether it has enough token. @@ -65,10 +87,12 @@ func (l *Limiter) Release(label string) { } // Update is used to update Ratelimiter with Options -func (l *Limiter) Update(label string, opts ...Option) { +func (l *Limiter) Update(label string, opts ...Option) UpdateStatus { + var status UpdateStatus for _, opt := range opts { - opt(label, l) + status |= opt(label, l) } + return status } // GetQPSLimiterStatus returns the status of a given label's QPS limiter. @@ -81,7 +105,7 @@ func (l *Limiter) GetQPSLimiterStatus(label string) (limit rate.Limit, burst int } // DeleteQPSLimiter deletes QPS limiter of given label -func (l *Limiter) DeleteQPSLimiter(label string) { +func (l *Limiter) deleteQPSLimiter(label string) { l.qpsLimiter.Delete(label) } @@ -95,7 +119,7 @@ func (l *Limiter) GetConcurrencyLimiterStatus(label string) (limit uint64, curre } // DeleteConcurrencyLimiter deletes concurrency limiter of given label -func (l *Limiter) DeleteConcurrencyLimiter(label string) { +func (l *Limiter) deleteConcurrencyLimiter(label string) { l.concurrencyLimiter.Delete(label) } diff --git a/pkg/ratelimit/limiter_test.go b/pkg/ratelimit/limiter_test.go index bd095543a05..b41953bce17 100644 --- a/pkg/ratelimit/limiter_test.go +++ b/pkg/ratelimit/limiter_test.go @@ -71,7 +71,7 @@ func (s *testRatelimiterSuite) TestUpdateConcurrencyLimiter(c *C) { limiter.Release(label) } - limiter.DeleteConcurrencyLimiter(label) + limiter.deleteConcurrencyLimiter(label) failedCount = 0 successCount = 0 for i := 0; i < 15; i++ { @@ -99,7 +99,7 @@ func (s *testRatelimiterSuite) TestBlockList(c *C) { } c.Assert(limiter.IsInAllowList(label), Equals, true) - UpdateQPSLimiter(rate.Every(time.Second), 1)(label, limiter) + UpdateQPSLimiter(float64(rate.Every(time.Second)), 1)(label, limiter) for i := 0; i < 10; i++ { c.Assert(limiter.Allow(label), Equals, true) } @@ -107,7 +107,7 @@ func (s *testRatelimiterSuite) TestBlockList(c *C) { func (s *testRatelimiterSuite) TestUpdateQPSLimiter(c *C) { c.Parallel() - opts := []Option{UpdateQPSLimiter(rate.Every(time.Second), 1)} + opts := []Option{UpdateQPSLimiter(float64(rate.Every(time.Second)), 1)} limiter := NewLimiter() label := "test" @@ -144,7 +144,7 @@ func (s *testRatelimiterSuite) TestUpdateQPSLimiter(c *C) { } } time.Sleep(time.Second) - limiter.DeleteQPSLimiter(label) + limiter.deleteQPSLimiter(label) for i := 0; i < 10; i++ { c.Assert(limiter.Allow(label), Equals, true) } @@ -155,7 +155,7 @@ func (s *testRatelimiterSuite) TestUpdateQPSLimiter(c *C) { func (s *testRatelimiterSuite) TestQPSLimiter(c *C) { c.Parallel() - opts := []Option{UpdateQPSLimiter(rate.Every(3*time.Second), 100)} + opts := []Option{UpdateQPSLimiter(float64(rate.Every(3*time.Second)), 100)} limiter := NewLimiter() label := "test" @@ -184,9 +184,12 @@ func (s *testRatelimiterSuite) TestQPSLimiter(c *C) { func (s *testRatelimiterSuite) TestTwoLimiters(c *C) { c.Parallel() - opts := []Option{UpdateQPSLimiter(100, 100), - UpdateConcurrencyLimiter(100), + cfg := DimensionConfig{ + QPS: 100, + QPSBrust: 100, + ConcurrencyLimit: 100, } + opts := []Option{UpdateDimensionConfig(cfg)} limiter := NewLimiter() label := "test" @@ -217,7 +220,7 @@ func (s *testRatelimiterSuite) TestTwoLimiters(c *C) { for i := 0; i < 100; i++ { limiter.Release(label) } - limiter.Update(label, UpdateQPSLimiter(rate.Every(10*time.Second), 1)) + limiter.Update(label, UpdateQPSLimiter(float64(rate.Every(10*time.Second)), 1)) wg.Add(100) for i := 0; i < 100; i++ { go CountRateLimiterHandleResult(limiter, label, &successCount, &failedCount, &lock, &wg) diff --git a/pkg/ratelimit/option.go b/pkg/ratelimit/option.go index af98eddb827..d10d19f6130 100644 --- a/pkg/ratelimit/option.go +++ b/pkg/ratelimit/option.go @@ -16,39 +16,112 @@ package ratelimit import "golang.org/x/time/rate" +// UpdateStatus is flags for updating limiter config. +type UpdateStatus uint32 + +// Flags for limiter. +const ( + // QPSNoChange shows that limiter's config isn't changed. + QPSNoChange UpdateStatus = 1 << iota + // QPSChanged shows that limiter's config is changed and not deleted. + QPSChanged + // QPSDeleted shows that limiter's config is deleted. + QPSDeleted + // ConcurrencyNoChange shows that limiter's config isn't changed. + ConcurrencyNoChange + // ConcurrencyChanged shows that limiter's config is changed and not deleted. + ConcurrencyChanged + // ConcurrencyDeleted shows that limiter's config is deleted. + ConcurrencyDeleted + // InAllowList shows that limiter's config isn't changed because it is in in allow list. + InAllowList + // Ignore shows that the status can be ignored when initiated + Ignore +) + // Option is used to create a limiter with the optional settings. // these setting is used to add a kind of limiter for a service -type Option func(string, *Limiter) +type Option func(string, *Limiter) UpdateStatus // AddLabelAllowList adds a label into allow list. // It means the given label will not be limited func AddLabelAllowList() Option { - return func(label string, l *Limiter) { + return func(label string, l *Limiter) UpdateStatus { l.labelAllowList[label] = struct{}{} + return Ignore + } +} + +func updateConcurrencyConfig(l *Limiter, label string, limit uint64) UpdateStatus { + l.configMux.Lock() + defer l.configMux.Unlock() + + cfg := l.labelConfig[label] + if cfg.ConcurrencyLimit == limit { + return ConcurrencyNoChange + } + cfg.ConcurrencyLimit = limit + l.labelConfig[label] = cfg + if limit < 1 { + l.deleteConcurrencyLimiter(label) + return ConcurrencyDeleted + } + if limiter, exist := l.concurrencyLimiter.LoadOrStore(label, newConcurrencyLimiter(limit)); exist { + limiter.(*concurrencyLimiter).setLimit(limit) } + return ConcurrencyChanged +} + +func updateQPSConfig(l *Limiter, label string, limit float64, burst int) UpdateStatus { + l.configMux.Lock() + defer l.configMux.Unlock() + + cfg := l.labelConfig[label] + if cfg.QPS == limit && cfg.QPSBrust == burst { + return QPSNoChange + } + cfg.QPS = limit + cfg.QPSBrust = burst + l.labelConfig[label] = cfg + if limit <= 0 || burst < 1 { + l.deleteQPSLimiter(label) + return QPSDeleted + } + if limiter, exist := l.qpsLimiter.LoadOrStore(label, NewRateLimiter(limit, burst)); exist { + limiter.(*RateLimiter).SetLimit(rate.Limit(limit)) + limiter.(*RateLimiter).SetBurst(burst) + } + return QPSChanged } // UpdateConcurrencyLimiter creates a concurrency limiter for a given label if it doesn't exist. func UpdateConcurrencyLimiter(limit uint64) Option { - return func(label string, l *Limiter) { + return func(label string, l *Limiter) UpdateStatus { if _, allow := l.labelAllowList[label]; allow { - return - } - if limiter, exist := l.concurrencyLimiter.LoadOrStore(label, newConcurrencyLimiter(limit)); exist { - limiter.(*concurrencyLimiter).setLimit(limit) + return InAllowList } + return updateConcurrencyConfig(l, label, limit) } } // UpdateQPSLimiter creates a QPS limiter for a given label if it doesn't exist. -func UpdateQPSLimiter(limit rate.Limit, burst int) Option { - return func(label string, l *Limiter) { +func UpdateQPSLimiter(limit float64, burst int) Option { + return func(label string, l *Limiter) UpdateStatus { if _, allow := l.labelAllowList[label]; allow { - return + return InAllowList } - if limiter, exist := l.qpsLimiter.LoadOrStore(label, NewRateLimiter(float64(limit), burst)); exist { - limiter.(*RateLimiter).SetLimit(limit) - limiter.(*RateLimiter).SetBurst(burst) + return updateQPSConfig(l, label, limit, burst) + } +} + +// UpdateDimensionConfig creates QPS limiter and concurrency limiter for a given label by config if it doesn't exist. +func UpdateDimensionConfig(cfg DimensionConfig) Option { + return func(label string, l *Limiter) UpdateStatus { + if _, allow := l.labelAllowList[label]; allow { + return InAllowList } + status := updateQPSConfig(l, label, cfg.QPS, cfg.QPSBrust) + status |= updateConcurrencyConfig(l, label, cfg.ConcurrencyLimit) + return status } } From b9bcfbf6016eccc969e570527a26fb8c3edc3787 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Sun, 24 Apr 2022 16:54:59 +0800 Subject: [PATCH 02/56] add rate-limit config Signed-off-by: Cabinfever_B --- server/api/config.go | 17 ++++++++++++++--- server/api/config_test.go | 10 +++++++--- server/config/config.go | 14 +++++++++++++- server/config/persist_options.go | 5 +++++ 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/server/api/config.go b/server/api/config.go index f4d653aad20..55c874cdf85 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -30,6 +30,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/apiutil" "github.com/tikv/pd/pkg/logutil" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/server" "github.com/tikv/pd/server/config" "github.com/unrolled/render" @@ -141,6 +142,16 @@ func (h *confHandler) SetConfig(w http.ResponseWriter, r *http.Request) { h.rd.JSON(w, http.StatusOK, "The config is updated.") } +func updateRateLimitConfig(svr *server.Server, key, label string, value ratelimit.DimensionConfig) error { + cfg := svr.GetConfig() + rateLimitCfg := ratelimit.NewLimiterConfig() + for label, item := range cfg.PDServerCfg.RateLimitConfig { + rateLimitCfg[label] = item + } + rateLimitCfg[label] = value + return updatePDServerConfig(svr, cfg, key, &rateLimitCfg) +} + func (h *confHandler) updateConfig(cfg *config.Config, key string, value interface{}) error { kp := strings.Split(key, ".") switch kp[0] { @@ -157,7 +168,7 @@ func (h *confHandler) updateConfig(cfg *config.Config, key string, value interfa } return h.updateReplicationModeConfig(cfg, kp[1:], value) case "pd-server": - return h.updatePDServerConfig(cfg, kp[len(kp)-1], value) + return updatePDServerConfig(h.svr, cfg, kp[len(kp)-1], value) case "log": return h.updateLogLevel(kp, value) case "cluster-version": @@ -255,7 +266,7 @@ func (h *confHandler) updateReplicationModeConfig(config *config.Config, key []s return err } -func (h *confHandler) updatePDServerConfig(config *config.Config, key string, value interface{}) error { +func updatePDServerConfig(svr *server.Server, config *config.Config, key string, value interface{}) error { data, err := json.Marshal(map[string]interface{}{key: value}) if err != nil { return err @@ -271,7 +282,7 @@ func (h *confHandler) updatePDServerConfig(config *config.Config, key string, va } if updated { - err = h.svr.SetPDServerConfig(config.PDServerCfg) + err = svr.SetPDServerConfig(config.PDServerCfg) } return err } diff --git a/server/api/config_test.go b/server/api/config_test.go index 6ccf7da8dad..d62c76c0a3b 100644 --- a/server/api/config_test.go +++ b/server/api/config_test.go @@ -303,9 +303,10 @@ func (s *testConfigSuite) TestConfigPDServer(c *C) { c.Assert(sc.MaxResetTSGap.Duration, Equals, 24*time.Hour) c.Assert(sc.EnableAudit, Equals, false) - // test update enable-audit + // test update enable-audit and enable-rate-limit ms = map[string]interface{}{ - "enable-audit": true, + "enable-audit": "true", + "enable-rate-limit": "true", } postData, err = json.Marshal(ms) c.Assert(err, IsNil) @@ -313,8 +314,10 @@ func (s *testConfigSuite) TestConfigPDServer(c *C) { sc = &config.PDServerConfig{} c.Assert(readJSON(testDialClient, addrGet, sc), IsNil) c.Assert(sc.EnableAudit, Equals, true) + c.Assert(sc.EnableRateLimit, Equals, true) ms = map[string]interface{}{ - "enable-audit": false, + "enable-audit": "false", + "enable-rate-limit": "false", } postData, err = json.Marshal(ms) c.Assert(err, IsNil) @@ -322,6 +325,7 @@ func (s *testConfigSuite) TestConfigPDServer(c *C) { sc = &config.PDServerConfig{} c.Assert(readJSON(testDialClient, addrGet, sc), IsNil) c.Assert(sc.EnableAudit, Equals, false) + c.Assert(sc.EnableRateLimit, Equals, false) } var ttlConfig = map[string]interface{}{ diff --git a/server/config/config.go b/server/config/config.go index 69305d0b30d..7337fa7ca30 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -32,6 +32,7 @@ import ( "github.com/tikv/pd/pkg/grpcutil" "github.com/tikv/pd/pkg/logutil" "github.com/tikv/pd/pkg/metricutil" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/typeutil" "github.com/tikv/pd/server/core/storelimit" "github.com/tikv/pd/server/versioninfo" @@ -233,6 +234,7 @@ const ( defaultMaxResetTSGap = 24 * time.Hour defaultMinResolvedTSPersistenceInterval = 0 defaultEnableAuditMiddleware = false + defaultEnableRateLimitMiddleware = false defaultKeyType = "table" defaultStrictlyMatchLabel = false @@ -1111,7 +1113,11 @@ type PDServerConfig struct { // MinResolvedTSPersistenceInterval is the interval to save the min resolved ts. MinResolvedTSPersistenceInterval typeutil.Duration `toml:"min-resolved-ts-persistence-interval" json:"min-resolved-ts-persistence-interval"` // EnableAudit controls the switch of the audit middleware - EnableAudit bool `toml:"enable-audit" json:"enable-audit"` + EnableAudit bool `toml:"enable-audit" json:"enable-audit,string"` + // EnableRateLimit controls the switch of the rate limit middleware + EnableRateLimit bool `toml:"enable-rate-limit" json:"enable-rate-limit,string"` + // RateLimitConfig is the config of rate limit middleware + RateLimitConfig ratelimit.LimiterConfig `toml:"rate-limit-config" json:"rate-limit-config"` } func (c *PDServerConfig) adjust(meta *configMetaData) error { @@ -1140,6 +1146,12 @@ func (c *PDServerConfig) adjust(meta *configMetaData) error { if !meta.IsDefined("enable-audit") { c.EnableAudit = defaultEnableAuditMiddleware } + if !meta.IsDefined("enable-rate-limit") { + c.EnableRateLimit = defaultEnableRateLimitMiddleware + } + if !meta.IsDefined("rate-limit-config") { + c.RateLimitConfig = ratelimit.NewLimiterConfig() + } c.migrateConfigurationFromFile(meta) return c.Validate() } diff --git a/server/config/persist_options.go b/server/config/persist_options.go index 039ff81a1af..daa66b5c75f 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -449,6 +449,11 @@ func (o *PersistOptions) IsAuditEnabled() bool { return o.GetPDServerConfig().EnableAudit } +// IsRateLimitEnabled returns whether rate limit middleware is enabled +func (o *PersistOptions) IsRateLimitEnabled() bool { + return o.GetPDServerConfig().EnableRateLimit +} + // GetKeyType is to get key type. func (o *PersistOptions) GetKeyType() core.KeyType { return core.StringToKeyType(o.GetPDServerConfig().KeyType) From 1a936044adf5dccbf045c9d5c1d47b982790a435 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Sun, 24 Apr 2022 17:20:55 +0800 Subject: [PATCH 03/56] fix test Signed-off-by: Cabinfever_B --- tests/server/api/api_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index 41216064c2f..5909a48f790 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -153,7 +153,7 @@ func (s *testMiddlewareSuite) TestRequestInfoMiddleware(c *C) { leader := s.cluster.GetServer(s.cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": true, + "enable-audit": "true", } data, err := json.Marshal(input) c.Assert(err, IsNil) @@ -181,7 +181,7 @@ func (s *testMiddlewareSuite) TestRequestInfoMiddleware(c *C) { c.Assert(resp.Header.Get("ip"), Equals, "127.0.0.1") input = map[string]interface{}{ - "enable-audit": false, + "enable-audit": "false", } data, err = json.Marshal(input) c.Assert(err, IsNil) @@ -206,7 +206,7 @@ func BenchmarkDoRequestWithServiceMiddleware(b *testing.B) { cluster.WaitLeader() leader := cluster.GetServer(cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": true, + "enable-audit": "true", } data, _ := json.Marshal(input) req, _ := http.NewRequest("POST", leader.GetAddr()+"/pd/api/v1/config", bytes.NewBuffer(data)) @@ -229,7 +229,7 @@ func BenchmarkDoRequestWithoutServiceMiddleware(b *testing.B) { cluster.WaitLeader() leader := cluster.GetServer(cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": false, + "enable-audit": "false", } data, _ := json.Marshal(input) req, _ := http.NewRequest("POST", leader.GetAddr()+"/pd/api/v1/config", bytes.NewBuffer(data)) @@ -256,7 +256,7 @@ func (s *testMiddlewareSuite) TestAuditMiddleware(c *C) { leader := s.cluster.GetServer(s.cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": true, + "enable-audit": "true", } data, err := json.Marshal(input) c.Assert(err, IsNil) @@ -290,7 +290,7 @@ func (s *testMiddlewareSuite) TestAuditMiddleware(c *C) { c.Assert(resp.Header.Get("audit-label"), Equals, "test") input = map[string]interface{}{ - "enable-audit": false, + "enable-audit": "false", } data, err = json.Marshal(input) c.Assert(err, IsNil) @@ -315,7 +315,7 @@ func (s *testMiddlewareSuite) TestAuditMiddleware(c *C) { func (s *testMiddlewareSuite) TestAuditPrometheusBackend(c *C) { leader := s.cluster.GetServer(s.cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": true, + "enable-audit": "true", } data, err := json.Marshal(input) c.Assert(err, IsNil) @@ -341,7 +341,7 @@ func (s *testMiddlewareSuite) TestAuditPrometheusBackend(c *C) { c.Assert(strings.Contains(output, "pd_service_audit_handling_seconds_count{component=\"anonymous\",method=\"HTTP\",service=\"GetTrend\"} 1"), Equals, true) input = map[string]interface{}{ - "enable-audit": false, + "enable-audit": "false", } data, err = json.Marshal(input) c.Assert(err, IsNil) @@ -361,7 +361,7 @@ func (s *testMiddlewareSuite) TestAuditLocalLogBackend(c *C) { log.ReplaceGlobals(lg, p) leader := s.cluster.GetServer(s.cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": true, + "enable-audit": "true", } data, err := json.Marshal(input) c.Assert(err, IsNil) @@ -393,7 +393,7 @@ func BenchmarkDoRequestWithLocalLogAudit(b *testing.B) { cluster.WaitLeader() leader := cluster.GetServer(cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": true, + "enable-audit": "true", } data, _ := json.Marshal(input) req, _ := http.NewRequest("POST", leader.GetAddr()+"/pd/api/v1/config", bytes.NewBuffer(data)) @@ -416,7 +416,7 @@ func BenchmarkDoRequestWithoutLocalLogAudit(b *testing.B) { cluster.WaitLeader() leader := cluster.GetServer(cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": false, + "enable-audit": "false", } data, _ := json.Marshal(input) req, _ := http.NewRequest("POST", leader.GetAddr()+"/pd/api/v1/config", bytes.NewBuffer(data)) From cefebc9ea7db43e78f6a41759f4af6ade44e899e Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Sun, 24 Apr 2022 17:29:53 +0800 Subject: [PATCH 04/56] remove future code Signed-off-by: Cabinfever_B --- server/api/config.go | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/server/api/config.go b/server/api/config.go index 55c874cdf85..f4d653aad20 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -30,7 +30,6 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/apiutil" "github.com/tikv/pd/pkg/logutil" - "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/server" "github.com/tikv/pd/server/config" "github.com/unrolled/render" @@ -142,16 +141,6 @@ func (h *confHandler) SetConfig(w http.ResponseWriter, r *http.Request) { h.rd.JSON(w, http.StatusOK, "The config is updated.") } -func updateRateLimitConfig(svr *server.Server, key, label string, value ratelimit.DimensionConfig) error { - cfg := svr.GetConfig() - rateLimitCfg := ratelimit.NewLimiterConfig() - for label, item := range cfg.PDServerCfg.RateLimitConfig { - rateLimitCfg[label] = item - } - rateLimitCfg[label] = value - return updatePDServerConfig(svr, cfg, key, &rateLimitCfg) -} - func (h *confHandler) updateConfig(cfg *config.Config, key string, value interface{}) error { kp := strings.Split(key, ".") switch kp[0] { @@ -168,7 +157,7 @@ func (h *confHandler) updateConfig(cfg *config.Config, key string, value interfa } return h.updateReplicationModeConfig(cfg, kp[1:], value) case "pd-server": - return updatePDServerConfig(h.svr, cfg, kp[len(kp)-1], value) + return h.updatePDServerConfig(cfg, kp[len(kp)-1], value) case "log": return h.updateLogLevel(kp, value) case "cluster-version": @@ -266,7 +255,7 @@ func (h *confHandler) updateReplicationModeConfig(config *config.Config, key []s return err } -func updatePDServerConfig(svr *server.Server, config *config.Config, key string, value interface{}) error { +func (h *confHandler) updatePDServerConfig(config *config.Config, key string, value interface{}) error { data, err := json.Marshal(map[string]interface{}{key: value}) if err != nil { return err @@ -282,7 +271,7 @@ func updatePDServerConfig(svr *server.Server, config *config.Config, key string, } if updated { - err = svr.SetPDServerConfig(config.PDServerCfg) + err = h.svr.SetPDServerConfig(config.PDServerCfg) } return err } From 826ef7ff09cd2a94201b1fd711775ec78f4c765b Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 25 Apr 2022 11:30:57 +0800 Subject: [PATCH 05/56] add rate-limit config presist reload Signed-off-by: Cabinfever_B --- server/server.go | 16 +++++ tests/server/config/config_test.go | 106 +++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 tests/server/config/config_test.go diff --git a/server/server.go b/server/server.go index 0e60b706123..267cac571ac 100644 --- a/server/server.go +++ b/server/server.go @@ -45,6 +45,7 @@ import ( "github.com/tikv/pd/pkg/etcdutil" "github.com/tikv/pd/pkg/grpcutil" "github.com/tikv/pd/pkg/logutil" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/systimemon" "github.com/tikv/pd/pkg/typeutil" "github.com/tikv/pd/server/cluster" @@ -155,6 +156,7 @@ type Server struct { // the corresponding forwarding TSO channel. tsoDispatcher sync.Map /* Store as map[string]chan *tsoRequest */ + serviceRateLimiter *ratelimit.Limiter serviceLabels map[string][]apiutil.AccessPath apiServiceLabelMap map[apiutil.AccessPath]string @@ -255,6 +257,7 @@ func CreateServer(ctx context.Context, cfg *config.Config, serviceBuilders ...Ha audit.NewPrometheusHistogramBackend(serviceAuditHistogram, false), } s.serviceAuditBackendLabels = make(map[string]*audit.BackendLabels) + s.serviceRateLimiter = ratelimit.NewLimiter() s.serviceLabels = make(map[string][]apiutil.AccessPath) s.apiServiceLabelMap = make(map[apiutil.AccessPath]string) @@ -1171,6 +1174,11 @@ func (s *Server) SetServiceAuditBackendLabels(serviceLabel string, labels []stri s.serviceAuditBackendLabels[serviceLabel] = &audit.BackendLabels{Labels: labels} } +// GetServiceRateLimiter is used to get rate limiter +func (s *Server) GetServiceRateLimiter() *ratelimit.Limiter { + return s.serviceRateLimiter +} + // GetClusterStatus gets cluster status. func (s *Server) GetClusterStatus() (*cluster.Status, error) { s.cluster.Lock() @@ -1415,6 +1423,7 @@ func (s *Server) reloadConfigFromKV() error { if err != nil { return err } + s.loadRateLimitConfig() switchableStorage, ok := s.storage.(interface { SwitchToRegionStorage() SwitchToDefaultStorage() @@ -1432,6 +1441,13 @@ func (s *Server) reloadConfigFromKV() error { return nil } +func (s *Server) loadRateLimitConfig() { + cfg := s.GetConfig().PDServerCfg.RateLimitConfig + for key, value := range cfg { + s.serviceRateLimiter.Update(key, ratelimit.UpdateDimensionConfig(value)) + } +} + // ReplicateFileToMember is used to synchronize state to a member. // Each member will write `data` to a local file named `name`. // For security reason, data should be in JSON format. diff --git a/tests/server/config/config_test.go b/tests/server/config/config_test.go new file mode 100644 index 00000000000..c60f0138ede --- /dev/null +++ b/tests/server/config/config_test.go @@ -0,0 +1,106 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "testing" + + . "github.com/pingcap/check" + "github.com/tikv/pd/pkg/ratelimit" + "github.com/tikv/pd/pkg/testutil" + "github.com/tikv/pd/server" + "github.com/tikv/pd/tests" +) + +// dialClient used to dial http request. +var dialClient = &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + }, +} + +func Test(t *testing.T) { + TestingT(t) +} + +var _ = Suite(&testConfigPresistSuite{}) + +type testConfigPresistSuite struct { + cleanup func() + cluster *tests.TestCluster +} + +func (s *testConfigPresistSuite) SetUpSuite(c *C) { + ctx, cancel := context.WithCancel(context.Background()) + server.EnableZap = true + s.cleanup = cancel + cluster, err := tests.NewTestCluster(ctx, 3) + c.Assert(err, IsNil) + c.Assert(cluster.RunInitialServers(), IsNil) + c.Assert(cluster.WaitLeader(), Not(HasLen), 0) + s.cluster = cluster +} + +func (s *testConfigPresistSuite) TearDownSuite(c *C) { + s.cleanup() + s.cluster.Destroy() +} + +func (s *testConfigPresistSuite) TestRateLimitConfigReload(c *C) { + leader := s.cluster.GetServer(s.cluster.GetLeader()) + + c.Assert(leader.GetServer().GetConfig().PDServerCfg.RateLimitConfig, HasLen, 0) + limitCfg := ratelimit.NewLimiterConfig() + limitCfg["GetRegions"] = ratelimit.DimensionConfig{QPS: 1} + + input := map[string]interface{}{ + "enable-rate-limit": "true", + "rate-limit-config": limitCfg, + } + data, err := json.Marshal(input) + c.Assert(err, IsNil) + req, _ := http.NewRequest("POST", leader.GetAddr()+"/pd/api/v1/config", bytes.NewBuffer(data)) + resp, err := dialClient.Do(req) + c.Assert(err, IsNil) + resp.Body.Close() + c.Assert(leader.GetServer().GetPersistOptions().IsRateLimitEnabled(), Equals, true) + c.Assert(leader.GetServer().GetConfig().PDServerCfg.RateLimitConfig, HasLen, 1) + + oldLeaderName := leader.GetServer().Name() + leader.GetServer().GetMember().ResignEtcdLeader(leader.GetServer().Context(), oldLeaderName, "") + mustWaitLeader(c, s.cluster.GetServers()) + leader = s.cluster.GetServer(s.cluster.GetLeader()) + + c.Assert(leader.GetServer().GetPersistOptions().IsRateLimitEnabled(), Equals, true) + c.Assert(leader.GetServer().GetConfig().PDServerCfg.RateLimitConfig, HasLen, 1) +} + +func mustWaitLeader(c *C, svrs map[string]*tests.TestServer) *server.Server { + var leader *server.Server + testutil.WaitUntil(c, func() bool { + for _, s := range svrs { + if !s.GetServer().IsClosed() && s.GetServer().GetMember().IsLeader() { + leader = s.GetServer() + return true + } + } + return false + }) + return leader +} From 8f24203642cc296cf076908c7e49115f0e4ad410 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 25 Apr 2022 13:03:12 +0800 Subject: [PATCH 06/56] add rate-limit config api Signed-off-by: Cabinfever_B --- pkg/apiutil/apiutil.go | 26 ++++++ server/api/admin.go | 99 +++++++++++++++++++++++ server/api/admin_test.go | 170 +++++++++++++++++++++++++++++++++++++++ server/api/config.go | 17 +++- server/api/util.go | 4 + server/server.go | 18 +++++ 6 files changed, 331 insertions(+), 3 deletions(-) diff --git a/pkg/apiutil/apiutil.go b/pkg/apiutil/apiutil.go index 5a8bfea605f..4b7ccc2076e 100644 --- a/pkg/apiutil/apiutil.go +++ b/pkg/apiutil/apiutil.go @@ -224,6 +224,16 @@ func PostJSON(client *http.Client, url string, data []byte, checkOpts ...func([] return doJSON(client, req, checkOpts...) } +// PostJSONIgnoreRespStatus is used to send the POST request to a specific URL and ignore resp status to test +func PostJSONIgnoreRespStatus(client *http.Client, url string, data []byte, checkOpts ...func([]byte, int)) error { + req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + return doJSONIgnoreRespStatus(client, req, checkOpts...) +} + // GetJSON is used to send GET requst to specific url func GetJSON(client *http.Client, url string, data []byte, checkOpts ...func([]byte, int)) error { req, err := http.NewRequest(http.MethodGet, url, bytes.NewBuffer(data)) @@ -251,3 +261,19 @@ func doJSON(client *http.Client, req *http.Request, checkOpts ...func([]byte, in } return nil } + +func doJSONIgnoreRespStatus(client *http.Client, req *http.Request, checkOpts ...func([]byte, int)) error { + resp, err := client.Do(req) + if err != nil { + return errors.WithStack(err) + } + defer resp.Body.Close() + res, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + for _, opt := range checkOpts { + opt(res, resp.StatusCode) + } + return nil +} diff --git a/server/api/admin.go b/server/api/admin.go index 36419234fc9..d5ede337867 100644 --- a/server/api/admin.go +++ b/server/api/admin.go @@ -16,12 +16,14 @@ package api import ( "encoding/json" + "fmt" "io" "net/http" "strconv" "github.com/gorilla/mux" "github.com/tikv/pd/pkg/apiutil" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/server" "github.com/unrolled/render" ) @@ -138,3 +140,100 @@ func (h *adminHandler) UpdateWaitAsyncTime(w http.ResponseWriter, r *http.Reques cluster.GetReplicationMode().UpdateMemberWaitAsyncTime(memberID) h.rd.JSON(w, http.StatusOK, nil) } + +// @Tags admin +// @Summary switch ratelimit middleware +// @Param enable query string true "enable" Enums(true, false) +// @Produce json +// @Success 200 {string} string "" +// @Failure 400 {string} string "" +// @Router /admin/ratelimit/config [POST] +func (h *adminHandler) SetRatelimitConfig(w http.ResponseWriter, r *http.Request) { + var input map[string]interface{} + if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &input); err != nil { + return + } + typeStr, ok := input["type"].(string) + if !ok { + h.rd.JSON(w, http.StatusBadRequest, "The type is empty.") + return + } + var serviceLabel string + switch typeStr { + case "label": + serviceLabel, ok = input["label"].(string) + if !ok || len(serviceLabel) == 0 { + h.rd.JSON(w, http.StatusBadRequest, "The label is empty.") + return + } + if len(h.svr.GetServiceLabels(serviceLabel)) == 0 { + h.rd.JSON(w, http.StatusBadRequest, "There is no label matched.") + return + } + case "path": + method, _ := input["method"].(string) + path, ok := input["path"].(string) + if !ok || len(path) == 0 { + h.rd.JSON(w, http.StatusBadRequest, "The path is empty.") + return + } + serviceLabel = h.svr.GetAPIAccessServiceLabel(apiutil.NewAccessPath(path, method)) + if len(serviceLabel) == 0 { + h.rd.JSON(w, http.StatusBadRequest, "There is no label matched.") + return + } + default: + h.rd.JSON(w, http.StatusBadRequest, "The type is invalid.") + return + } + if h.svr.IsInRateLimitBlockList(serviceLabel) { + h.rd.JSON(w, http.StatusBadRequest, "This service is in block list.") + return + } + cfg := h.svr.GetConfig().PDServerCfg.RateLimitConfig[serviceLabel] + // update concurrency limiter + concurrencyUpdatedFlag := "Concurrency limiter is not changed." + concurrencyFloat, okc := input["concurrency"].(float64) + if okc { + concurrency := uint64(concurrencyFloat) + cfg.ConcurrencyLimit = concurrency + } + // update qps rate limiter + qpsRateUpdatedFlag := "QPS rate limiter is not changed." + qps, okq := input["qps"].(float64) + if okq { + brust := 0 + if qps > 0 { + if int(qps) > 1 { + brust = int(qps) + } else { + brust = 1 + } + } + cfg.QPS = qps + cfg.QPSBrust = brust + } + status := h.svr.UpdateServiceRateLimiter(serviceLabel, ratelimit.UpdateDimensionConfig(cfg)) + switch { + case status&ratelimit.QPSChanged != 0: + qpsRateUpdatedFlag = "QPS rate limiter is changed." + case status&ratelimit.QPSDeleted != 0: + qpsRateUpdatedFlag = "QPS rate limiter is deleted." + } + switch { + case status&ratelimit.ConcurrencyChanged != 0: + concurrencyUpdatedFlag = "Concurrency limiter is changed." + case status&ratelimit.ConcurrencyDeleted != 0: + concurrencyUpdatedFlag = "Concurrency limiter is deleted." + } + if !okc && !okq { + h.rd.JSON(w, http.StatusOK, "No changed.") + } else { + err := updateRateLimitConfig(h.svr, "rate-limit-config", serviceLabel, cfg) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + } else { + h.rd.JSON(w, http.StatusOK, fmt.Sprintf("%s %s", concurrencyUpdatedFlag, qpsRateUpdatedFlag)) + } + } +} diff --git a/server/api/admin_test.go b/server/api/admin_test.go index 8744d09092e..2234372d681 100644 --- a/server/api/admin_test.go +++ b/server/api/admin_test.go @@ -192,3 +192,173 @@ func (s *testServiceSuite) SetUpSuite(c *C) { func (s *testServiceSuite) TearDownSuite(c *C) { s.cleanup() } + +func (s *testServiceSuite) TestUpdateRateLimitConfig(c *C) { + urlPrefix := fmt.Sprintf("%s%s/api/v1/admin/ratelimit/config", s.svr.GetAddr(), apiPrefix) + + // test empty type + input := make(map[string]interface{}) + input["type"] = 123 + jsonBody, err := json.Marshal(input) + c.Assert(err, IsNil) + + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(string(res), Equals, "\"The type is empty.\"\n") + c.Assert(code, Equals, http.StatusBadRequest) + }) + c.Assert(err, IsNil) + // test invalid type + input = make(map[string]interface{}) + input["type"] = "url" + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(string(res), Equals, "\"The type is invalid.\"\n") + c.Assert(code, Equals, http.StatusBadRequest) + }) + c.Assert(err, IsNil) + + // test empty label + input = make(map[string]interface{}) + input["type"] = "label" + input["label"] = "" + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(string(res), Equals, "\"The label is empty.\"\n") + c.Assert(code, Equals, http.StatusBadRequest) + }) + c.Assert(err, IsNil) + // test no label matched + input = make(map[string]interface{}) + input["type"] = "label" + input["label"] = "TestLabel" + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(string(res), Equals, "\"There is no label matched.\"\n") + c.Assert(code, Equals, http.StatusBadRequest) + }) + c.Assert(err, IsNil) + + // test empty path + input = make(map[string]interface{}) + input["type"] = "path" + input["path"] = "" + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(string(res), Equals, "\"The path is empty.\"\n") + c.Assert(code, Equals, http.StatusBadRequest) + }) + c.Assert(err, IsNil) + + // test path but no label matched + input = make(map[string]interface{}) + input["type"] = "path" + input["path"] = "/pd/api/v1/test" + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(string(res), Equals, "\"There is no label matched.\"\n") + c.Assert(code, Equals, http.StatusBadRequest) + }) + c.Assert(err, IsNil) + + // no change + input = make(map[string]interface{}) + input["type"] = "label" + input["label"] = "GetHealthStatus" + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(string(res), Equals, "\"No changed.\"\n") + c.Assert(code, Equals, http.StatusOK) + }) + c.Assert(err, IsNil) + + // change concurrency + input = make(map[string]interface{}) + input["type"] = "path" + input["path"] = "/pd/api/v1/health" + input["method"] = "GET" + input["concurrency"] = 100 + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(strings.Contains(string(res), "Concurrency limiter is changed."), Equals, true) + c.Assert(code, Equals, http.StatusOK) + }) + c.Assert(err, IsNil) + input["concurrency"] = 0 + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(strings.Contains(string(res), "Concurrency limiter is deleted."), Equals, true) + c.Assert(code, Equals, http.StatusOK) + }) + c.Assert(err, IsNil) + + // change qps + input = make(map[string]interface{}) + input["type"] = "path" + input["path"] = "/pd/api/v1/health" + input["method"] = "GET" + input["qps"] = 100 + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(strings.Contains(string(res), "QPS rate limiter is changed."), Equals, true) + c.Assert(code, Equals, http.StatusOK) + }) + c.Assert(err, IsNil) + input["qps"] = -1 + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(strings.Contains(string(res), "QPS rate limiter is deleted."), Equals, true) + c.Assert(code, Equals, http.StatusOK) + }) + c.Assert(err, IsNil) + + // change both + input = make(map[string]interface{}) + input["type"] = "path" + input["path"] = "/pd/api/v1/debug/pprof/profile" + input["qps"] = 100 + input["concurrency"] = 100 + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(string(res), Equals, "\"Concurrency limiter is changed. QPS rate limiter is changed.\"\n") + c.Assert(code, Equals, http.StatusOK) + }) + c.Assert(err, IsNil) + + // block list + input = make(map[string]interface{}) + input["type"] = "label" + input["label"] = "SetRatelimitConfig" + input["qps"] = 100 + input["concurrency"] = 100 + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(string(res), Equals, "\"This service is in block list.\"\n") + c.Assert(code, Equals, http.StatusBadRequest) + }) + c.Assert(err, IsNil) +} diff --git a/server/api/config.go b/server/api/config.go index f4d653aad20..55c874cdf85 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -30,6 +30,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/apiutil" "github.com/tikv/pd/pkg/logutil" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/server" "github.com/tikv/pd/server/config" "github.com/unrolled/render" @@ -141,6 +142,16 @@ func (h *confHandler) SetConfig(w http.ResponseWriter, r *http.Request) { h.rd.JSON(w, http.StatusOK, "The config is updated.") } +func updateRateLimitConfig(svr *server.Server, key, label string, value ratelimit.DimensionConfig) error { + cfg := svr.GetConfig() + rateLimitCfg := ratelimit.NewLimiterConfig() + for label, item := range cfg.PDServerCfg.RateLimitConfig { + rateLimitCfg[label] = item + } + rateLimitCfg[label] = value + return updatePDServerConfig(svr, cfg, key, &rateLimitCfg) +} + func (h *confHandler) updateConfig(cfg *config.Config, key string, value interface{}) error { kp := strings.Split(key, ".") switch kp[0] { @@ -157,7 +168,7 @@ func (h *confHandler) updateConfig(cfg *config.Config, key string, value interfa } return h.updateReplicationModeConfig(cfg, kp[1:], value) case "pd-server": - return h.updatePDServerConfig(cfg, kp[len(kp)-1], value) + return updatePDServerConfig(h.svr, cfg, kp[len(kp)-1], value) case "log": return h.updateLogLevel(kp, value) case "cluster-version": @@ -255,7 +266,7 @@ func (h *confHandler) updateReplicationModeConfig(config *config.Config, key []s return err } -func (h *confHandler) updatePDServerConfig(config *config.Config, key string, value interface{}) error { +func updatePDServerConfig(svr *server.Server, config *config.Config, key string, value interface{}) error { data, err := json.Marshal(map[string]interface{}{key: value}) if err != nil { return err @@ -271,7 +282,7 @@ func (h *confHandler) updatePDServerConfig(config *config.Config, key string, va } if updated { - err = h.svr.SetPDServerConfig(config.PDServerCfg) + err = svr.SetPDServerConfig(config.PDServerCfg) } return err } diff --git a/server/api/util.go b/server/api/util.go index 0cea83498a6..e9b18e6e85e 100644 --- a/server/api/util.go +++ b/server/api/util.go @@ -95,6 +95,10 @@ func postJSON(client *http.Client, url string, data []byte, checkOpts ...func([] return apiutil.PostJSON(client, url, data, checkOpts...) } +func postJSONIgnoreRespStatus(client *http.Client, url string, data []byte, checkOpts ...func([]byte, int)) error { + return apiutil.PostJSONIgnoreRespStatus(client, url, data, checkOpts...) +} + func getJSON(client *http.Client, url string, data []byte, checkOpts ...func([]byte, int)) error { return apiutil.GetJSON(client, url, data, checkOpts...) } diff --git a/server/server.go b/server/server.go index 0e60b706123..c5ae729b598 100644 --- a/server/server.go +++ b/server/server.go @@ -45,6 +45,7 @@ import ( "github.com/tikv/pd/pkg/etcdutil" "github.com/tikv/pd/pkg/grpcutil" "github.com/tikv/pd/pkg/logutil" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/systimemon" "github.com/tikv/pd/pkg/typeutil" "github.com/tikv/pd/server/cluster" @@ -155,6 +156,7 @@ type Server struct { // the corresponding forwarding TSO channel. tsoDispatcher sync.Map /* Store as map[string]chan *tsoRequest */ + serviceRateLimiter *ratelimit.Limiter serviceLabels map[string][]apiutil.AccessPath apiServiceLabelMap map[apiutil.AccessPath]string @@ -254,6 +256,7 @@ func CreateServer(ctx context.Context, cfg *config.Config, serviceBuilders ...Ha audit.NewLocalLogBackend(true), audit.NewPrometheusHistogramBackend(serviceAuditHistogram, false), } + s.serviceRateLimiter = ratelimit.NewLimiter() s.serviceAuditBackendLabels = make(map[string]*audit.BackendLabels) s.serviceLabels = make(map[string][]apiutil.AccessPath) s.apiServiceLabelMap = make(map[apiutil.AccessPath]string) @@ -1171,6 +1174,21 @@ func (s *Server) SetServiceAuditBackendLabels(serviceLabel string, labels []stri s.serviceAuditBackendLabels[serviceLabel] = &audit.BackendLabels{Labels: labels} } +// GetServiceRateLimiter is used to get rate limiter +func (s *Server) GetServiceRateLimiter() *ratelimit.Limiter { + return s.serviceRateLimiter +} + +// IsInRateLimitBlockList returns whethis given service label is in block lost +func (s *Server) IsInRateLimitBlockList(serviceLabel string) bool { + return s.serviceRateLimiter.IsInAllowList(serviceLabel) +} + +// UpdateServiceRateLimiter is used to update RateLimiter +func (s *Server) UpdateServiceRateLimiter(serviceLabel string, opts ...ratelimit.Option) ratelimit.UpdateStatus { + return s.serviceRateLimiter.Update(serviceLabel, opts...) +} + // GetClusterStatus gets cluster status. func (s *Server) GetClusterStatus() (*cluster.Status, error) { s.cluster.Lock() From 176d73b603f8611ebee7de61ed7fbb478ae5ffb6 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 25 Apr 2022 13:18:52 +0800 Subject: [PATCH 07/56] fix test Signed-off-by: Cabinfever_B --- server/api/admin_test.go | 4 ++++ server/api/router.go | 1 + 2 files changed, 5 insertions(+) diff --git a/server/api/admin_test.go b/server/api/admin_test.go index 2234372d681..5ca6f6773a3 100644 --- a/server/api/admin_test.go +++ b/server/api/admin_test.go @@ -23,6 +23,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/server" "github.com/tikv/pd/server/core" ) @@ -347,6 +348,9 @@ func (s *testServiceSuite) TestUpdateRateLimitConfig(c *C) { }) c.Assert(err, IsNil) + limiter := s.svr.GetServiceRateLimiter() + limiter.Update("SetRatelimitConfig", ratelimit.AddLabelAllowList()) + // block list input = make(map[string]interface{}) input["type"] = "label" diff --git a/server/api/router.go b/server/api/router.go index cae5c2a70ee..3ddf942c992 100644 --- a/server/api/router.go +++ b/server/api/router.go @@ -312,6 +312,7 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(clusterRouter, "/admin/reset-ts", adminHandler.ResetTS, setMethods("POST"), setAuditBackend(localLog)) registerFunc(apiRouter, "/admin/persist-file/{file_name}", adminHandler.SavePersistFile, setMethods("POST"), setAuditBackend(localLog)) registerFunc(clusterRouter, "/admin/replication_mode/wait-async", adminHandler.UpdateWaitAsyncTime, setMethods("POST"), setAuditBackend(localLog)) + registerFunc(apiRouter, "/admin/ratelimit/config", adminHandler.SetRatelimitConfig, setMethods("POST"), setAuditBackend(localLog)) logHandler := newLogHandler(svr, rd) registerFunc(apiRouter, "/admin/log", logHandler.SetLogLevel, setMethods("POST"), setAuditBackend(localLog)) From 0d73c530be821d6484421c6f2b1927b7bc34b85d Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 25 Apr 2022 13:38:10 +0800 Subject: [PATCH 08/56] fix comment Signed-off-by: Cabinfever_B --- server/api/admin_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/admin_test.go b/server/api/admin_test.go index 5ca6f6773a3..b99d3568ad2 100644 --- a/server/api/admin_test.go +++ b/server/api/admin_test.go @@ -351,7 +351,7 @@ func (s *testServiceSuite) TestUpdateRateLimitConfig(c *C) { limiter := s.svr.GetServiceRateLimiter() limiter.Update("SetRatelimitConfig", ratelimit.AddLabelAllowList()) - // block list + // Allow list input = make(map[string]interface{}) input["type"] = "label" input["label"] = "SetRatelimitConfig" From 3318100728ae5897d708e88a1b649e400ebc95d4 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 25 Apr 2022 15:18:21 +0800 Subject: [PATCH 09/56] add test Signed-off-by: Cabinfever_B --- pkg/ratelimit/limiter_test.go | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/pkg/ratelimit/limiter_test.go b/pkg/ratelimit/limiter_test.go index b41953bce17..d73664d6e36 100644 --- a/pkg/ratelimit/limiter_test.go +++ b/pkg/ratelimit/limiter_test.go @@ -34,9 +34,8 @@ func (s *testRatelimiterSuite) TestUpdateConcurrencyLimiter(c *C) { limiter := NewLimiter() label := "test" - for _, opt := range opts { - opt(label, limiter) - } + status := limiter.Update(label, opts...) + c.Assert(status&ConcurrencyChanged != 0, IsTrue) var lock sync.Mutex successCount, failedCount := 0, 0 var wg sync.WaitGroup @@ -57,7 +56,11 @@ func (s *testRatelimiterSuite) TestUpdateConcurrencyLimiter(c *C) { c.Assert(limit, Equals, uint64(10)) c.Assert(current, Equals, uint64(0)) - limiter.Update(label, UpdateConcurrencyLimiter(5)) + status = limiter.Update(label, UpdateConcurrencyLimiter(10)) + c.Assert(status&ConcurrencyNoChange != 0, IsTrue) + + status = limiter.Update(label, UpdateConcurrencyLimiter(5)) + c.Assert(status&ConcurrencyChanged != 0, IsTrue) failedCount = 0 successCount = 0 for i := 0; i < 15; i++ { @@ -71,7 +74,8 @@ func (s *testRatelimiterSuite) TestUpdateConcurrencyLimiter(c *C) { limiter.Release(label) } - limiter.deleteConcurrencyLimiter(label) + status = limiter.Update(label, UpdateConcurrencyLimiter(0)) + c.Assert(status&ConcurrencyDeleted != 0, IsTrue) failedCount = 0 successCount = 0 for i := 0; i < 15; i++ { @@ -99,7 +103,8 @@ func (s *testRatelimiterSuite) TestBlockList(c *C) { } c.Assert(limiter.IsInAllowList(label), Equals, true) - UpdateQPSLimiter(float64(rate.Every(time.Second)), 1)(label, limiter) + status := UpdateQPSLimiter(float64(rate.Every(time.Second)), 1)(label, limiter) + c.Assert(status&InAllowList != 0, Equals, true) for i := 0; i < 10; i++ { c.Assert(limiter.Allow(label), Equals, true) } @@ -111,9 +116,8 @@ func (s *testRatelimiterSuite) TestUpdateQPSLimiter(c *C) { limiter := NewLimiter() label := "test" - for _, opt := range opts { - opt(label, limiter) - } + status := limiter.Update(label, opts...) + c.Assert(status&QPSChanged != 0, IsTrue) var lock sync.Mutex successCount, failedCount := 0, 0 @@ -130,7 +134,11 @@ func (s *testRatelimiterSuite) TestUpdateQPSLimiter(c *C) { c.Assert(limit, Equals, rate.Limit(1)) c.Assert(burst, Equals, 1) - limiter.Update(label, UpdateQPSLimiter(5, 5)) + status = limiter.Update(label, UpdateQPSLimiter(float64(rate.Every(time.Second)), 1)) + c.Assert(status&QPSNoChange != 0, IsTrue) + + status = limiter.Update(label, UpdateQPSLimiter(5, 5)) + c.Assert(status&QPSChanged != 0, IsTrue) limit, burst = limiter.GetQPSLimiterStatus(label) c.Assert(limit, Equals, rate.Limit(5)) c.Assert(burst, Equals, 5) @@ -144,7 +152,9 @@ func (s *testRatelimiterSuite) TestUpdateQPSLimiter(c *C) { } } time.Sleep(time.Second) - limiter.deleteQPSLimiter(label) + + status = limiter.Update(label, UpdateQPSLimiter(0, 0)) + c.Assert(status&QPSDeleted != 0, IsTrue) for i := 0; i < 10; i++ { c.Assert(limiter.Allow(label), Equals, true) } From 4e2a840a3444870cf7d7b459083d3f11d4a57f2b Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 25 Apr 2022 15:56:19 +0800 Subject: [PATCH 10/56] fix comment Signed-off-by: Cabinfever_B --- pkg/ratelimit/limiter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ratelimit/limiter.go b/pkg/ratelimit/limiter.go index ccc93037ea0..b934b67f07b 100644 --- a/pkg/ratelimit/limiter.go +++ b/pkg/ratelimit/limiter.go @@ -104,7 +104,7 @@ func (l *Limiter) GetQPSLimiterStatus(label string) (limit rate.Limit, burst int return 0, 0 } -// DeleteQPSLimiter deletes QPS limiter of given label +// deleteQPSLimiter deletes QPS limiter of a given label func (l *Limiter) deleteQPSLimiter(label string) { l.qpsLimiter.Delete(label) } @@ -118,7 +118,7 @@ func (l *Limiter) GetConcurrencyLimiterStatus(label string) (limit uint64, curre return 0, 0 } -// DeleteConcurrencyLimiter deletes concurrency limiter of given label +// deleteConcurrencyLimiter deletes concurrency limiter of a given label func (l *Limiter) deleteConcurrencyLimiter(label string) { l.concurrencyLimiter.Delete(label) } From 456d5043336dde01fb95eb1f8adc6dd3b74c7d50 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 25 Apr 2022 16:27:00 +0800 Subject: [PATCH 11/56] add test Signed-off-by: Cabinfever_B --- server/api/admin_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/server/api/admin_test.go b/server/api/admin_test.go index b99d3568ad2..51d918facc0 100644 --- a/server/api/admin_test.go +++ b/server/api/admin_test.go @@ -323,6 +323,22 @@ func (s *testServiceSuite) TestUpdateRateLimitConfig(c *C) { c.Assert(code, Equals, http.StatusOK) }) c.Assert(err, IsNil) + + input = make(map[string]interface{}) + input["type"] = "path" + input["path"] = "/pd/api/v1/health" + input["method"] = "GET" + input["qps"] = 0.3 + jsonBody, err = json.Marshal(input) + c.Assert(err, IsNil) + err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, + func(res []byte, code int) { + c.Assert(strings.Contains(string(res), "QPS rate limiter is changed."), Equals, true) + c.Assert(code, Equals, http.StatusOK) + }) + c.Assert(err, IsNil) + c.Assert(s.svr.GetConfig().PDServerCfg.RateLimitConfig["GetHealthStatus"].QPSBrust, Equals, 1) + input["qps"] = -1 jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) From 049c4d67db490e1943624ac4a4f952a8568dbc53 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 26 Apr 2022 01:28:37 +0800 Subject: [PATCH 12/56] fix typo Signed-off-by: Cabinfever_B --- pkg/ratelimit/option.go | 6 +++--- server/config/persist_options.go | 5 ----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/pkg/ratelimit/option.go b/pkg/ratelimit/option.go index d10d19f6130..79e07fa736b 100644 --- a/pkg/ratelimit/option.go +++ b/pkg/ratelimit/option.go @@ -77,11 +77,11 @@ func updateQPSConfig(l *Limiter, label string, limit float64, burst int) UpdateS defer l.configMux.Unlock() cfg := l.labelConfig[label] - if cfg.QPS == limit && cfg.QPSBrust == burst { + if cfg.QPS == limit && cfg.QPSBurst == burst { return QPSNoChange } cfg.QPS = limit - cfg.QPSBrust = burst + cfg.QPSBurst = burst l.labelConfig[label] = cfg if limit <= 0 || burst < 1 { l.deleteQPSLimiter(label) @@ -120,7 +120,7 @@ func UpdateDimensionConfig(cfg DimensionConfig) Option { if _, allow := l.labelAllowList[label]; allow { return InAllowList } - status := updateQPSConfig(l, label, cfg.QPS, cfg.QPSBrust) + status := updateQPSConfig(l, label, cfg.QPS, cfg.QPSBurst) status |= updateConcurrencyConfig(l, label, cfg.ConcurrencyLimit) return status } diff --git a/server/config/persist_options.go b/server/config/persist_options.go index daa66b5c75f..039ff81a1af 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -449,11 +449,6 @@ func (o *PersistOptions) IsAuditEnabled() bool { return o.GetPDServerConfig().EnableAudit } -// IsRateLimitEnabled returns whether rate limit middleware is enabled -func (o *PersistOptions) IsRateLimitEnabled() bool { - return o.GetPDServerConfig().EnableRateLimit -} - // GetKeyType is to get key type. func (o *PersistOptions) GetKeyType() core.KeyType { return core.StringToKeyType(o.GetPDServerConfig().KeyType) From 1b183a046f3b64d61792dcd5abe966120a9daf2e Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 26 Apr 2022 10:25:34 +0800 Subject: [PATCH 13/56] fix typo Signed-off-by: Cabinfever_B --- pkg/ratelimit/limiter.go | 2 +- pkg/ratelimit/limiter_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ratelimit/limiter.go b/pkg/ratelimit/limiter.go index b934b67f07b..45661f38216 100644 --- a/pkg/ratelimit/limiter.go +++ b/pkg/ratelimit/limiter.go @@ -32,7 +32,7 @@ func NewLimiterConfig() LimiterConfig { type DimensionConfig struct { // qps conifg QPS float64 - QPSBrust int + QPSBurst int // concurrency config ConcurrencyLimit uint64 } diff --git a/pkg/ratelimit/limiter_test.go b/pkg/ratelimit/limiter_test.go index d73664d6e36..f1ae1662fa4 100644 --- a/pkg/ratelimit/limiter_test.go +++ b/pkg/ratelimit/limiter_test.go @@ -196,7 +196,7 @@ func (s *testRatelimiterSuite) TestTwoLimiters(c *C) { c.Parallel() cfg := DimensionConfig{ QPS: 100, - QPSBrust: 100, + QPSBurst: 100, ConcurrencyLimit: 100, } opts := []Option{UpdateDimensionConfig(cfg)} From 900df005eb744f39bcfbf97574492691ff591a87 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Thu, 28 Apr 2022 14:14:19 +0800 Subject: [PATCH 14/56] add service config Signed-off-by: Cabinfever_B --- server/api/config.go | 32 +++++++++++++++++++++ server/api/config_test.go | 49 ++++++++++++++++++-------------- server/api/router.go | 1 + server/config/config.go | 32 +++++++++++++++++---- server/config/config_test.go | 4 +++ server/config/persist_options.go | 16 ++++++++++- server/server.go | 22 ++++++++++++++ tests/server/api/api_test.go | 4 +-- 8 files changed, 131 insertions(+), 29 deletions(-) diff --git a/server/api/config.go b/server/api/config.go index f6bf8e59799..fcd8309d6ec 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -151,6 +151,8 @@ func (h *confHandler) updateConfig(cfg *config.Config, key string, value interfa return errors.Errorf("cannot update config prefix %s", kp[0]) } return h.updateReplicationModeConfig(cfg, kp[1:], value) + case "service": + return h.updateServiceConfig(cfg, kp[len(kp)-1], value) case "pd-server": return h.updatePDServerConfig(cfg, kp[len(kp)-1], value) case "log": @@ -250,6 +252,27 @@ func (h *confHandler) updateReplicationModeConfig(config *config.Config, key []s return err } +func (h *confHandler) updateServiceConfig(config *config.Config, key string, value interface{}) error { + data, err := json.Marshal(map[string]interface{}{key: value}) + if err != nil { + return err + } + + updated, found, err := mergeConfig(&config.ServiceCfg, data) + if err != nil { + return err + } + + if !found { + return errors.Errorf("config item %s not found", key) + } + + if updated { + err = h.svr.SetServiceConfig(config.ServiceCfg) + } + return err +} + func (h *confHandler) updatePDServerConfig(config *config.Config, key string, value interface{}) error { data, err := json.Marshal(map[string]interface{}{key: value}) if err != nil { @@ -535,6 +558,15 @@ func (h *confHandler) SetReplicationModeConfig(w http.ResponseWriter, r *http.Re h.rd.JSON(w, http.StatusOK, "The replication mode config is updated.") } +// @Tags config +// @Summary Get Service config. +// @Produce json +// @Success 200 {object} config.Service +// @Router /config/service [get] +func (h *confHandler) GetServiceConfig(w http.ResponseWriter, r *http.Request) { + h.rd.JSON(w, http.StatusOK, h.svr.GetServiceConfig()) +} + // @Tags config // @Summary Get PD server config. // @Produce json diff --git a/server/api/config_test.go b/server/api/config_test.go index 51d2998a743..efec411526a 100644 --- a/server/api/config_test.go +++ b/server/api/config_test.go @@ -279,6 +279,34 @@ func (s *testConfigSuite) TestConfigDefault(c *C) { c.Assert(defaultCfg.PDServerCfg.MetricStorage, Equals, "") } +func (s *testConfigSuite) TestConfigService(c *C) { + addrGet := fmt.Sprintf("%s/config/service", s.urlPrefix) + sc := &config.ServiceConfig{} + c.Assert(readJSON(testDialClient, addrGet, sc), IsNil) + c.Assert(sc.EnableAudit, Equals, false) + + addrPost := fmt.Sprintf("%s/config", s.urlPrefix) + // test update enable-audit + ms := map[string]interface{}{ + "enable-audit": "true", + } + postData, err := json.Marshal(ms) + c.Assert(err, IsNil) + c.Assert(postJSON(testDialClient, addrPost, postData), IsNil) + sc = &config.ServiceConfig{} + c.Assert(readJSON(testDialClient, addrGet, sc), IsNil) + c.Assert(sc.EnableAudit, Equals, true) + ms = map[string]interface{}{ + "enable-audit": "false", + } + postData, err = json.Marshal(ms) + c.Assert(err, IsNil) + c.Assert(postJSON(testDialClient, addrPost, postData), IsNil) + sc = &config.ServiceConfig{} + c.Assert(readJSON(testDialClient, addrGet, sc), IsNil) + c.Assert(sc.EnableAudit, Equals, false) +} + func (s *testConfigSuite) TestConfigPDServer(c *C) { addrPost := fmt.Sprintf("%s/config", s.urlPrefix) @@ -301,27 +329,6 @@ func (s *testConfigSuite) TestConfigPDServer(c *C) { c.Assert(sc.FlowRoundByDigit, Equals, int(3)) c.Assert(sc.MinResolvedTSPersistenceInterval, Equals, typeutil.NewDuration(0)) c.Assert(sc.MaxResetTSGap.Duration, Equals, 24*time.Hour) - c.Assert(sc.EnableAudit, Equals, false) - - // test update enable-audit - ms = map[string]interface{}{ - "enable-audit": true, - } - postData, err = json.Marshal(ms) - c.Assert(err, IsNil) - c.Assert(postJSON(testDialClient, addrPost, postData), IsNil) - sc = &config.PDServerConfig{} - c.Assert(readJSON(testDialClient, addrGet, sc), IsNil) - c.Assert(sc.EnableAudit, Equals, true) - ms = map[string]interface{}{ - "enable-audit": false, - } - postData, err = json.Marshal(ms) - c.Assert(err, IsNil) - c.Assert(postJSON(testDialClient, addrPost, postData), IsNil) - sc = &config.PDServerConfig{} - c.Assert(readJSON(testDialClient, addrGet, sc), IsNil) - c.Assert(sc.EnableAudit, Equals, false) } var ttlConfig = map[string]interface{}{ diff --git a/server/api/router.go b/server/api/router.go index fbf04446f3a..b2937a2478b 100644 --- a/server/api/router.go +++ b/server/api/router.go @@ -148,6 +148,7 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(apiRouter, "/config/default", confHandler.GetDefaultConfig, setMethods("GET")) registerFunc(apiRouter, "/config/schedule", confHandler.GetScheduleConfig, setMethods("GET")) registerFunc(apiRouter, "/config/schedule", confHandler.SetScheduleConfig, setMethods("POST"), setAuditBackend(localLog)) + registerFunc(apiRouter, "/config/service", confHandler.GetServiceConfig, setMethods("GET")) registerFunc(apiRouter, "/config/pd-server", confHandler.GetPDServerConfig, setMethods("GET")) registerFunc(apiRouter, "/config/replicate", confHandler.GetReplicationConfig, setMethods("GET")) registerFunc(apiRouter, "/config/replicate", confHandler.SetReplicationConfig, setMethods("POST"), setAuditBackend(localLog)) diff --git a/server/config/config.go b/server/config/config.go index 69305d0b30d..f5cd46b3434 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -107,6 +107,8 @@ type Config struct { Replication ReplicationConfig `toml:"replication" json:"replication"` + ServiceCfg ServiceConfig `toml:"service" json:"service"` + PDServerCfg PDServerConfig `toml:"pd-server" json:"pd-server"` ClusterVersion semver.Version `toml:"cluster-version" json:"cluster-version"` @@ -574,6 +576,10 @@ func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { return err } + if err := c.ServiceCfg.adjust(configMetaData.Child("service")); err != nil { + return err + } + if err := c.PDServerCfg.adjust(configMetaData.Child("pd-server")); err != nil { return err } @@ -1086,6 +1092,27 @@ func (c *ReplicationConfig) adjust(meta *configMetaData) error { return c.Validate() } +// ServiceConfig +type ServiceConfig struct { + // EnableAudit controls the switch of the audit middleware + EnableAudit bool `toml:"enable-audit" json:"enable-audit,string"` + // EnableRateLimit controls the switch of the rate limit middleware + EnableRateLimit bool `toml:"enable-rate-limit" json:"enable-rate-limit,string"` +} + +// Clone returns a cloned PD server config. +func (c *ServiceConfig) Clone() *ServiceConfig { + cfg := *c + return &cfg +} + +func (c *ServiceConfig) adjust(meta *configMetaData) error { + if !meta.IsDefined("enable-audit") { + c.EnableAudit = defaultEnableAuditMiddleware + } + return nil +} + // PDServerConfig is the configuration for pd server. // NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. type PDServerConfig struct { @@ -1110,8 +1137,6 @@ type PDServerConfig struct { FlowRoundByDigit int `toml:"flow-round-by-digit" json:"flow-round-by-digit"` // MinResolvedTSPersistenceInterval is the interval to save the min resolved ts. MinResolvedTSPersistenceInterval typeutil.Duration `toml:"min-resolved-ts-persistence-interval" json:"min-resolved-ts-persistence-interval"` - // EnableAudit controls the switch of the audit middleware - EnableAudit bool `toml:"enable-audit" json:"enable-audit"` } func (c *PDServerConfig) adjust(meta *configMetaData) error { @@ -1137,9 +1162,6 @@ func (c *PDServerConfig) adjust(meta *configMetaData) error { if !meta.IsDefined("min-resolved-ts-persistence-interval") { adjustDuration(&c.MinResolvedTSPersistenceInterval, defaultMinResolvedTSPersistenceInterval) } - if !meta.IsDefined("enable-audit") { - c.EnableAudit = defaultEnableAuditMiddleware - } c.migrateConfigurationFromFile(meta) return c.Validate() } diff --git a/server/config/config_test.go b/server/config/config_test.go index 686d7cf52a6..c83d7a03bcb 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -502,6 +502,10 @@ func (s *testConfigSuite) TestConfigClone(c *C) { replication.adjust(emptyConfigMetaData) c.Assert(replication.Clone(), DeepEquals, replication) + service := &ServiceConfig{} + service.adjust(emptyConfigMetaData) + c.Assert(service.Clone(), DeepEquals, service) + pdServer := &PDServerConfig{} pdServer.adjust(emptyConfigMetaData) c.Assert(pdServer.Clone(), DeepEquals, pdServer) diff --git a/server/config/persist_options.go b/server/config/persist_options.go index 039ff81a1af..a9c7c9453fa 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -46,6 +46,7 @@ type PersistOptions struct { ttl *cache.TTLString schedule atomic.Value replication atomic.Value + serviceConfig atomic.Value pdServerConfig atomic.Value replicationMode atomic.Value labelProperty atomic.Value @@ -57,6 +58,7 @@ func NewPersistOptions(cfg *Config) *PersistOptions { o := &PersistOptions{} o.schedule.Store(&cfg.Schedule) o.replication.Store(&cfg.Replication) + o.serviceConfig.Store(&cfg.ServiceCfg) o.pdServerConfig.Store(&cfg.PDServerCfg) o.replicationMode.Store(&cfg.ReplicationMode) o.labelProperty.Store(cfg.LabelProperty) @@ -85,6 +87,16 @@ func (o *PersistOptions) SetReplicationConfig(cfg *ReplicationConfig) { o.replication.Store(cfg) } +// GetServiceConfig returns pd server configurations. +func (o *PersistOptions) GetServiceConfig() *ServiceConfig { + return o.serviceConfig.Load().(*ServiceConfig) +} + +// SetPDServerConfig sets the PD configuration. +func (o *PersistOptions) SetServiceConfig(cfg *ServiceConfig) { + o.serviceConfig.Store(cfg) +} + // GetPDServerConfig returns pd server configurations. func (o *PersistOptions) GetPDServerConfig() *PDServerConfig { return o.pdServerConfig.Load().(*PDServerConfig) @@ -446,7 +458,7 @@ func (o *PersistOptions) GetLeaderSchedulePolicy() core.SchedulePolicy { // IsAuditEnabled returns whether audit middleware is enabled func (o *PersistOptions) IsAuditEnabled() bool { - return o.GetPDServerConfig().EnableAudit + return o.GetServiceConfig().EnableAudit } // GetKeyType is to get key type. @@ -598,6 +610,7 @@ func (o *PersistOptions) Persist(storage endpoint.ConfigStorage) error { cfg := &Config{ Schedule: *o.GetScheduleConfig(), Replication: *o.GetReplicationConfig(), + ServiceCfg: *o.GetServiceConfig(), PDServerCfg: *o.GetPDServerConfig(), ReplicationMode: *o.GetReplicationModeConfig(), LabelProperty: o.GetLabelPropertyConfig(), @@ -625,6 +638,7 @@ func (o *PersistOptions) Reload(storage endpoint.ConfigStorage) error { if isExist { o.schedule.Store(&cfg.Schedule) o.replication.Store(&cfg.Replication) + o.serviceConfig.Store(&cfg.ServiceCfg) o.pdServerConfig.Store(&cfg.PDServerCfg) o.replicationMode.Store(&cfg.ReplicationMode) o.labelProperty.Store(cfg.LabelProperty) diff --git a/server/server.go b/server/server.go index 0e60b706123..0aa0fcbc30f 100644 --- a/server/server.go +++ b/server/server.go @@ -812,6 +812,7 @@ func (s *Server) GetConfig() *config.Config { cfg := s.cfg.Clone() cfg.Schedule = *s.persistOptions.GetScheduleConfig().Clone() cfg.Replication = *s.persistOptions.GetReplicationConfig().Clone() + cfg.ServiceCfg = *s.persistOptions.GetServiceConfig().Clone() cfg.PDServerCfg = *s.persistOptions.GetPDServerConfig().Clone() cfg.ReplicationMode = *s.persistOptions.GetReplicationModeConfig() cfg.LabelProperty = s.persistOptions.GetLabelPropertyConfig().Clone() @@ -954,6 +955,27 @@ func (s *Server) SetReplicationConfig(cfg config.ReplicationConfig) error { return nil } +// GetServiceConfig gets the service config information. +func (s *Server) GetServiceConfig() *config.ServiceConfig { + return s.persistOptions.GetServiceConfig().Clone() +} + +// SetServiceConfig sets the service config. +func (s *Server) SetServiceConfig(cfg config.ServiceConfig) error { + old := s.persistOptions.GetServiceConfig() + s.persistOptions.SetServiceConfig(&cfg) + if err := s.persistOptions.Persist(s.storage); err != nil { + s.persistOptions.SetServiceConfig(old) + log.Error("failed to update Service config", + zap.Reflect("new", cfg), + zap.Reflect("old", old), + errs.ZapError(err)) + return err + } + log.Info("Service config is updated", zap.Reflect("new", cfg), zap.Reflect("old", old)) + return nil +} + // GetPDServerConfig gets the balance config information. func (s *Server) GetPDServerConfig() *config.PDServerConfig { return s.persistOptions.GetPDServerConfig().Clone() diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index 53306a01e8d..ca7424cc40b 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -254,7 +254,7 @@ func doTestRequest(srv *tests.TestServer) { func (s *testMiddlewareSuite) TestAuditPrometheusBackend(c *C) { leader := s.cluster.GetServer(s.cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": true, + "enable-audit": "true", } data, err := json.Marshal(input) c.Assert(err, IsNil) @@ -302,7 +302,7 @@ func (s *testMiddlewareSuite) TestAuditPrometheusBackend(c *C) { c.Assert(strings.Contains(output, "pd_service_audit_handling_seconds_count{component=\"anonymous\",method=\"HTTP\",service=\"GetTrend\"} 2"), Equals, true) input = map[string]interface{}{ - "enable-audit": false, + "enable-audit": "false", } data, err = json.Marshal(input) c.Assert(err, IsNil) From 133ab1510bb35404484fa14b1f8af29fe94f6638 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Thu, 28 Apr 2022 14:30:28 +0800 Subject: [PATCH 15/56] add service config Signed-off-by: Cabinfever_B --- server/api/config.go | 2 +- server/config/config.go | 4 +--- server/config/persist_options.go | 2 +- tests/server/api/api_test.go | 14 +++++++------- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/server/api/config.go b/server/api/config.go index fcd8309d6ec..cf240399c35 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -561,7 +561,7 @@ func (h *confHandler) SetReplicationModeConfig(w http.ResponseWriter, r *http.Re // @Tags config // @Summary Get Service config. // @Produce json -// @Success 200 {object} config.Service +// @Success 200 {object} config.ServiceConfig // @Router /config/service [get] func (h *confHandler) GetServiceConfig(w http.ResponseWriter, r *http.Request) { h.rd.JSON(w, http.StatusOK, h.svr.GetServiceConfig()) diff --git a/server/config/config.go b/server/config/config.go index f5cd46b3434..00de738dd8e 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -1092,12 +1092,10 @@ func (c *ReplicationConfig) adjust(meta *configMetaData) error { return c.Validate() } -// ServiceConfig +// ServiceConfig is the configuration for PD service such as HTTP API and gRPC. type ServiceConfig struct { // EnableAudit controls the switch of the audit middleware EnableAudit bool `toml:"enable-audit" json:"enable-audit,string"` - // EnableRateLimit controls the switch of the rate limit middleware - EnableRateLimit bool `toml:"enable-rate-limit" json:"enable-rate-limit,string"` } // Clone returns a cloned PD server config. diff --git a/server/config/persist_options.go b/server/config/persist_options.go index a9c7c9453fa..ee7d89c7b6b 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -92,7 +92,7 @@ func (o *PersistOptions) GetServiceConfig() *ServiceConfig { return o.serviceConfig.Load().(*ServiceConfig) } -// SetPDServerConfig sets the PD configuration. +// SetServiceConfig sets the PD configuration. func (o *PersistOptions) SetServiceConfig(cfg *ServiceConfig) { o.serviceConfig.Store(cfg) } diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index ca7424cc40b..ec9d284f2ab 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -153,7 +153,7 @@ func (s *testMiddlewareSuite) TestRequestInfoMiddleware(c *C) { leader := s.cluster.GetServer(s.cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": true, + "enable-audit": "true", } data, err := json.Marshal(input) c.Assert(err, IsNil) @@ -181,7 +181,7 @@ func (s *testMiddlewareSuite) TestRequestInfoMiddleware(c *C) { c.Assert(resp.Header.Get("ip"), Equals, "127.0.0.1") input = map[string]interface{}{ - "enable-audit": false, + "enable-audit": "false", } data, err = json.Marshal(input) c.Assert(err, IsNil) @@ -206,7 +206,7 @@ func BenchmarkDoRequestWithServiceMiddleware(b *testing.B) { cluster.WaitLeader() leader := cluster.GetServer(cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": true, + "enable-audit": "true", } data, _ := json.Marshal(input) req, _ := http.NewRequest("POST", leader.GetAddr()+"/pd/api/v1/config", bytes.NewBuffer(data)) @@ -229,7 +229,7 @@ func BenchmarkDoRequestWithoutServiceMiddleware(b *testing.B) { cluster.WaitLeader() leader := cluster.GetServer(cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": false, + "enable-audit": "false", } data, _ := json.Marshal(input) req, _ := http.NewRequest("POST", leader.GetAddr()+"/pd/api/v1/config", bytes.NewBuffer(data)) @@ -322,7 +322,7 @@ func (s *testMiddlewareSuite) TestAuditLocalLogBackend(c *C) { log.ReplaceGlobals(lg, p) leader := s.cluster.GetServer(s.cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": true, + "enable-audit": "true", } data, err := json.Marshal(input) c.Assert(err, IsNil) @@ -354,7 +354,7 @@ func BenchmarkDoRequestWithLocalLogAudit(b *testing.B) { cluster.WaitLeader() leader := cluster.GetServer(cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": true, + "enable-audit": "true", } data, _ := json.Marshal(input) req, _ := http.NewRequest("POST", leader.GetAddr()+"/pd/api/v1/config", bytes.NewBuffer(data)) @@ -377,7 +377,7 @@ func BenchmarkDoRequestWithoutLocalLogAudit(b *testing.B) { cluster.WaitLeader() leader := cluster.GetServer(cluster.GetLeader()) input := map[string]interface{}{ - "enable-audit": false, + "enable-audit": "false", } data, _ := json.Marshal(input) req, _ := http.NewRequest("POST", leader.GetAddr()+"/pd/api/v1/config", bytes.NewBuffer(data)) From 3a6be7c76a6beb6f98c6b677cf6829ba438941b3 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 18 May 2022 14:32:52 +0800 Subject: [PATCH 16/56] merge master Signed-off-by: Cabinfever_B --- server/api/config_test.go | 33 ---------------------- server/config/service_middleware_config.go | 22 +++++++++++++-- 2 files changed, 20 insertions(+), 35 deletions(-) diff --git a/server/api/config_test.go b/server/api/config_test.go index 67a5ca3bd19..271849ce223 100644 --- a/server/api/config_test.go +++ b/server/api/config_test.go @@ -283,39 +283,6 @@ func (s *testConfigSuite) TestConfigDefault(c *C) { c.Assert(defaultCfg.PDServerCfg.MetricStorage, Equals, "") } -func (s *testConfigSuite) TestConfigService(c *C) { - addrGet := fmt.Sprintf("%s/config/service", s.urlPrefix) - sc := &config.ServiceConfig{} - c.Assert(readJSON(testDialClient, addrGet, sc), IsNil) - c.Assert(sc.EnableAudit, Equals, false) - - addrPost := fmt.Sprintf("%s/config", s.urlPrefix) - - // test update enable-audit and enable-rate-limit - ms := map[string]interface{}{ - "enable-audit": "true", - "enable-rate-limit": "true", - } - postData, err := json.Marshal(ms) - c.Assert(err, IsNil) - c.Assert(postJSON(testDialClient, addrPost, postData), IsNil) - sc = &config.ServiceConfig{} - c.Assert(readJSON(testDialClient, addrGet, sc), IsNil) - c.Assert(sc.EnableAudit, Equals, true) - c.Assert(sc.EnableRateLimit, Equals, true) - ms = map[string]interface{}{ - "enable-audit": "false", - "enable-rate-limit": "false", - } - postData, err = json.Marshal(ms) - c.Assert(err, IsNil) - c.Assert(postJSON(testDialClient, addrPost, postData), IsNil) - sc = &config.ServiceConfig{} - c.Assert(readJSON(testDialClient, addrGet, sc), IsNil) - c.Assert(sc.EnableAudit, Equals, false) - c.Assert(sc.EnableRateLimit, Equals, false) -} - func (s *testConfigSuite) TestConfigPDServer(c *C) { addrPost := fmt.Sprintf("%s/config", s.urlPrefix) diff --git a/server/config/service_middleware_config.go b/server/config/service_middleware_config.go index d1b600ccaf2..225b95d9dd6 100644 --- a/server/config/service_middleware_config.go +++ b/server/config/service_middleware_config.go @@ -14,13 +14,17 @@ package config +import "github.com/tikv/pd/pkg/ratelimit" + const ( - defaultEnableAuditMiddleware = false + defaultEnableAuditMiddleware = false + defaultEnableRateLimitMiddleware = false ) // ServiceMiddlewareConfig is is the configuration for PD Service middleware. type ServiceMiddlewareConfig struct { - AuditConfig `json:"audit"` + AuditConfig `json:"audit"` + RateLimitConfig `json:"rate-limit"` } // NewServiceMiddlewareConfig returns a new service middleware config @@ -51,3 +55,17 @@ func (c *AuditConfig) Clone() *AuditConfig { cfg := *c return &cfg } + +// RateLimitConfig is the configuration for rate limit +type RateLimitConfig struct { + // EnableRateLimit controls the switch of the rate limit middleware + EnableRateLimit bool `json:"enable-rate-limit,string"` + // RateLimitConfig is the config of rate limit middleware + RateLimitConfig ratelimit.LimiterConfig `json:"rate-limit-config"` +} + +// Clone returns a cloned rate limit config. +func (c *RateLimitConfig) Clone() *RateLimitConfig { + cfg := *c + return &cfg +} From 124658c6c911c677736552be6118b2a8cfbeef59 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 18 May 2022 14:36:38 +0800 Subject: [PATCH 17/56] merge master Signed-off-by: Cabinfever_B --- server/api/config.go | 32 ---------------------- server/api/router.go | 1 - server/config/service_middleware_config.go | 6 +++- 3 files changed, 5 insertions(+), 34 deletions(-) diff --git a/server/api/config.go b/server/api/config.go index 9e825237b88..de87947b785 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -152,8 +152,6 @@ func (h *confHandler) updateConfig(cfg *config.Config, key string, value interfa return errors.Errorf("cannot update config prefix %s", kp[0]) } return h.updateReplicationModeConfig(cfg, kp[1:], value) - case "service": - return h.updateServiceConfig(cfg, kp[len(kp)-1], value) case "pd-server": return h.updatePDServerConfig(cfg, kp[len(kp)-1], value) case "log": @@ -230,27 +228,6 @@ func (h *confHandler) updateReplicationModeConfig(config *config.Config, key []s return err } -func (h *confHandler) updateServiceConfig(config *config.Config, key string, value interface{}) error { - data, err := json.Marshal(map[string]interface{}{key: value}) - if err != nil { - return err - } - - updated, found, err := mergeConfig(&config.ServiceCfg, data) - if err != nil { - return err - } - - if !found { - return errors.Errorf("config item %s not found", key) - } - - if updated { - err = h.svr.SetServiceConfig(config.ServiceCfg) - } - return err -} - func (h *confHandler) updatePDServerConfig(config *config.Config, key string, value interface{}) error { data, err := json.Marshal(map[string]interface{}{key: value}) if err != nil { @@ -527,15 +504,6 @@ func (h *confHandler) SetReplicationModeConfig(w http.ResponseWriter, r *http.Re h.rd.JSON(w, http.StatusOK, "The replication mode config is updated.") } -// @Tags config -// @Summary Get Service config. -// @Produce json -// @Success 200 {object} config.ServiceConfig -// @Router /config/service [get] -func (h *confHandler) GetServiceConfig(w http.ResponseWriter, r *http.Request) { - h.rd.JSON(w, http.StatusOK, h.svr.GetServiceConfig()) -} - // @Tags config // @Summary Get PD server config. // @Produce json diff --git a/server/api/router.go b/server/api/router.go index c98e504af31..b9eaeb4b090 100644 --- a/server/api/router.go +++ b/server/api/router.go @@ -148,7 +148,6 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(apiRouter, "/config/default", confHandler.GetDefaultConfig, setMethods("GET")) registerFunc(apiRouter, "/config/schedule", confHandler.GetScheduleConfig, setMethods("GET")) registerFunc(apiRouter, "/config/schedule", confHandler.SetScheduleConfig, setMethods("POST"), setAuditBackend(localLog)) - registerFunc(apiRouter, "/config/service", confHandler.GetServiceConfig, setMethods("GET")) registerFunc(apiRouter, "/config/pd-server", confHandler.GetPDServerConfig, setMethods("GET")) registerFunc(apiRouter, "/config/replicate", confHandler.GetReplicationConfig, setMethods("GET")) registerFunc(apiRouter, "/config/replicate", confHandler.SetReplicationConfig, setMethods("POST"), setAuditBackend(localLog)) diff --git a/server/config/service_middleware_config.go b/server/config/service_middleware_config.go index 225b95d9dd6..e568b2803ad 100644 --- a/server/config/service_middleware_config.go +++ b/server/config/service_middleware_config.go @@ -32,8 +32,12 @@ func NewServiceMiddlewareConfig() *ServiceMiddlewareConfig { audit := AuditConfig{ EnableAudit: defaultEnableAuditMiddleware, } + ratelimit := RateLimitConfig{ + EnableRateLimit: defaultEnableRateLimitMiddleware, + } cfg := &ServiceMiddlewareConfig{ - AuditConfig: audit, + AuditConfig: audit, + RateLimitConfig: ratelimit, } return cfg } From cadaebbbcd2afaf4d88714ba068c123edbce3b61 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 18 May 2022 14:53:39 +0800 Subject: [PATCH 18/56] merge master Signed-off-by: Cabinfever_B --- server/config/service_middleware_config.go | 28 +++------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/server/config/service_middleware_config.go b/server/config/service_middleware_config.go index e568b2803ad..d1b600ccaf2 100644 --- a/server/config/service_middleware_config.go +++ b/server/config/service_middleware_config.go @@ -14,17 +14,13 @@ package config -import "github.com/tikv/pd/pkg/ratelimit" - const ( - defaultEnableAuditMiddleware = false - defaultEnableRateLimitMiddleware = false + defaultEnableAuditMiddleware = false ) // ServiceMiddlewareConfig is is the configuration for PD Service middleware. type ServiceMiddlewareConfig struct { - AuditConfig `json:"audit"` - RateLimitConfig `json:"rate-limit"` + AuditConfig `json:"audit"` } // NewServiceMiddlewareConfig returns a new service middleware config @@ -32,12 +28,8 @@ func NewServiceMiddlewareConfig() *ServiceMiddlewareConfig { audit := AuditConfig{ EnableAudit: defaultEnableAuditMiddleware, } - ratelimit := RateLimitConfig{ - EnableRateLimit: defaultEnableRateLimitMiddleware, - } cfg := &ServiceMiddlewareConfig{ - AuditConfig: audit, - RateLimitConfig: ratelimit, + AuditConfig: audit, } return cfg } @@ -59,17 +51,3 @@ func (c *AuditConfig) Clone() *AuditConfig { cfg := *c return &cfg } - -// RateLimitConfig is the configuration for rate limit -type RateLimitConfig struct { - // EnableRateLimit controls the switch of the rate limit middleware - EnableRateLimit bool `json:"enable-rate-limit,string"` - // RateLimitConfig is the config of rate limit middleware - RateLimitConfig ratelimit.LimiterConfig `json:"rate-limit-config"` -} - -// Clone returns a cloned rate limit config. -func (c *RateLimitConfig) Clone() *RateLimitConfig { - cfg := *c - return &cfg -} From 464bc85876b07f5ec9f153377352d7f9d1ebac21 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 18 May 2022 17:17:56 +0800 Subject: [PATCH 19/56] merge master Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 26 ++++++++++++++++++- server/api/service_middleware_test.go | 6 ++++- server/config/service_middleware_config.go | 2 +- .../service_middleware_persist_options.go | 3 +++ server/server.go | 26 +++++++++++++++++-- tests/server/config/config_test.go | 14 +++++----- 6 files changed, 65 insertions(+), 12 deletions(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index c136f8fbf4e..0f41f8ae725 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -103,8 +103,11 @@ func (h *serviceMiddlewareHandler) SetServiceMiddlewareConfig(w http.ResponseWri func (h *serviceMiddlewareHandler) updateServiceMiddlewareConfig(cfg *config.ServiceMiddlewareConfig, key string, value interface{}) error { kp := strings.Split(key, ".") - if kp[0] == "audit" { + switch kp[0] { + case "audit": return h.updateAudit(cfg, kp[len(kp)-1], value) + case "rate-limit": + return h.updateRateLimit(cfg, kp[len(kp)-1], value) } return errors.Errorf("config prefix %s not found", kp[0]) } @@ -129,3 +132,24 @@ func (h *serviceMiddlewareHandler) updateAudit(config *config.ServiceMiddlewareC } return err } + +func (h *serviceMiddlewareHandler) updateRateLimit(config *config.ServiceMiddlewareConfig, key string, value interface{}) error { + data, err := json.Marshal(map[string]interface{}{key: value}) + if err != nil { + return err + } + + updated, found, err := mergeConfig(&config.RateLimitConfig, data) + if err != nil { + return err + } + + if !found { + return errors.Errorf("config item %s not found", key) + } + + if updated { + err = h.svr.SetRateLimitConfig(config.RateLimitConfig) + } + return err +} diff --git a/server/api/service_middleware_test.go b/server/api/service_middleware_test.go index 3d29b23a693..443998a25ff 100644 --- a/server/api/service_middleware_test.go +++ b/server/api/service_middleware_test.go @@ -51,7 +51,8 @@ func (s *testServiceMiddlewareSuite) TearDownSuite(c *C) { func (s *testServiceMiddlewareSuite) TestConfigAudit(c *C) { addr := fmt.Sprintf("%s/service-middleware/config", s.urlPrefix) ms := map[string]interface{}{ - "enable-audit": "true", + "enable-audit": "true", + "enable-rate-limit": "true", } postData, err := json.Marshal(ms) c.Assert(err, IsNil) @@ -59,8 +60,10 @@ func (s *testServiceMiddlewareSuite) TestConfigAudit(c *C) { sc := &config.ServiceMiddlewareConfig{} c.Assert(tu.ReadGetJSON(c, testDialClient, addr, sc), IsNil) c.Assert(sc.EnableAudit, Equals, true) + c.Assert(sc.EnableRateLimit, Equals, true) ms = map[string]interface{}{ "audit.enable-audit": "false", + "enable-rate-limit": "false", } postData, err = json.Marshal(ms) c.Assert(err, IsNil) @@ -68,6 +71,7 @@ func (s *testServiceMiddlewareSuite) TestConfigAudit(c *C) { sc = &config.ServiceMiddlewareConfig{} c.Assert(tu.ReadGetJSON(c, testDialClient, addr, sc), IsNil) c.Assert(sc.EnableAudit, Equals, false) + c.Assert(sc.EnableRateLimit, Equals, false) // test empty ms = map[string]interface{}{} diff --git a/server/config/service_middleware_config.go b/server/config/service_middleware_config.go index e568b2803ad..15dc8e1fd38 100644 --- a/server/config/service_middleware_config.go +++ b/server/config/service_middleware_config.go @@ -65,7 +65,7 @@ type RateLimitConfig struct { // EnableRateLimit controls the switch of the rate limit middleware EnableRateLimit bool `json:"enable-rate-limit,string"` // RateLimitConfig is the config of rate limit middleware - RateLimitConfig ratelimit.LimiterConfig `json:"rate-limit-config"` + LimiterConfig ratelimit.LimiterConfig `json:"limiter-config"` } // Clone returns a cloned rate limit config. diff --git a/server/config/service_middleware_persist_options.go b/server/config/service_middleware_persist_options.go index 8b8bd81489b..20f8c110a5f 100644 --- a/server/config/service_middleware_persist_options.go +++ b/server/config/service_middleware_persist_options.go @@ -52,14 +52,17 @@ func (o *ServiceMiddlewarePersistOptions) IsAuditEnabled() bool { return o.GetAuditConfig().EnableAudit } +// GetRateLimitConfig returns pd service middleware configurations. func (o *ServiceMiddlewarePersistOptions) GetRateLimitConfig() *RateLimitConfig { return o.rateLimit.Load().(*RateLimitConfig) } +// SetRateLimitConfig sets the PD service middleware configuration. func (o *ServiceMiddlewarePersistOptions) SetRateLimitConfig(cfg *RateLimitConfig) { o.rateLimit.Store(cfg) } +// IsRateLimitEnabled returns whether rate limit middleware is enabled func (o *ServiceMiddlewarePersistOptions) IsRateLimitEnabled() bool { return o.GetRateLimitConfig().EnableRateLimit } diff --git a/server/server.go b/server/server.go index 397245eb804..246d55bd5ea 100644 --- a/server/server.go +++ b/server/server.go @@ -817,7 +817,8 @@ func (s *Server) GetMembers() ([]*pdpb.Member, error) { // GetServiceMiddlewareConfig gets the service middleware config information. func (s *Server) GetServiceMiddlewareConfig() *config.ServiceMiddlewareConfig { cfg := s.serviceMiddlewareCfg.Clone() - cfg.AuditConfig = *s.serviceMiddlewarePersistOptions.GetAuditConfig() + cfg.AuditConfig = *s.serviceMiddlewarePersistOptions.GetAuditConfig().Clone() + cfg.RateLimitConfig = *s.serviceMiddlewarePersistOptions.GetRateLimitConfig().Clone() return cfg } @@ -989,6 +990,27 @@ func (s *Server) SetAuditConfig(cfg config.AuditConfig) error { return nil } +// GetRateLimitConfig gets the rate limit config information. +func (s *Server) GetRateLimitConfig() *config.RateLimitConfig { + return s.serviceMiddlewarePersistOptions.GetRateLimitConfig().Clone() +} + +// SetRateLimitConfig sets the rate limit config. +func (s *Server) SetRateLimitConfig(cfg config.RateLimitConfig) error { + old := s.serviceMiddlewarePersistOptions.GetRateLimitConfig() + s.serviceMiddlewarePersistOptions.SetRateLimitConfig(&cfg) + if err := s.serviceMiddlewarePersistOptions.Persist(s.storage); err != nil { + s.serviceMiddlewarePersistOptions.SetRateLimitConfig(old) + log.Error("failed to update Rate Limit config", + zap.Reflect("new", cfg), + zap.Reflect("old", old), + errs.ZapError(err)) + return err + } + log.Info("Rate Limit config is updated", zap.Reflect("new", cfg), zap.Reflect("old", old)) + return nil +} + // GetPDServerConfig gets the balance config information. func (s *Server) GetPDServerConfig() *config.PDServerConfig { return s.persistOptions.GetPDServerConfig().Clone() @@ -1485,7 +1507,7 @@ func (s *Server) reloadConfigFromKV() error { } func (s *Server) loadRateLimitConfig() { - cfg := s.serviceMiddlewarePersistOptions.GetRateLimitConfig().RateLimitConfig + cfg := s.serviceMiddlewarePersistOptions.GetRateLimitConfig().LimiterConfig for key, value := range cfg { s.serviceRateLimiter.Update(key, ratelimit.UpdateDimensionConfig(value)) } diff --git a/tests/server/config/config_test.go b/tests/server/config/config_test.go index c60f0138ede..7ce084243e3 100644 --- a/tests/server/config/config_test.go +++ b/tests/server/config/config_test.go @@ -65,30 +65,30 @@ func (s *testConfigPresistSuite) TearDownSuite(c *C) { func (s *testConfigPresistSuite) TestRateLimitConfigReload(c *C) { leader := s.cluster.GetServer(s.cluster.GetLeader()) - c.Assert(leader.GetServer().GetConfig().PDServerCfg.RateLimitConfig, HasLen, 0) + c.Assert(leader.GetServer().GetServiceMiddlewareConfig().RateLimitConfig.LimiterConfig, HasLen, 0) limitCfg := ratelimit.NewLimiterConfig() limitCfg["GetRegions"] = ratelimit.DimensionConfig{QPS: 1} input := map[string]interface{}{ "enable-rate-limit": "true", - "rate-limit-config": limitCfg, + "limiter-config": limitCfg, } data, err := json.Marshal(input) c.Assert(err, IsNil) - req, _ := http.NewRequest("POST", leader.GetAddr()+"/pd/api/v1/config", bytes.NewBuffer(data)) + req, _ := http.NewRequest("POST", leader.GetAddr()+"/pd/api/v1/service-middleware/config", bytes.NewBuffer(data)) resp, err := dialClient.Do(req) c.Assert(err, IsNil) resp.Body.Close() - c.Assert(leader.GetServer().GetPersistOptions().IsRateLimitEnabled(), Equals, true) - c.Assert(leader.GetServer().GetConfig().PDServerCfg.RateLimitConfig, HasLen, 1) + c.Assert(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled(), Equals, true) + c.Assert(leader.GetServer().GetServiceMiddlewarePersistOptions().GetRateLimitConfig().LimiterConfig, HasLen, 1) oldLeaderName := leader.GetServer().Name() leader.GetServer().GetMember().ResignEtcdLeader(leader.GetServer().Context(), oldLeaderName, "") mustWaitLeader(c, s.cluster.GetServers()) leader = s.cluster.GetServer(s.cluster.GetLeader()) - c.Assert(leader.GetServer().GetPersistOptions().IsRateLimitEnabled(), Equals, true) - c.Assert(leader.GetServer().GetConfig().PDServerCfg.RateLimitConfig, HasLen, 1) + c.Assert(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled(), Equals, true) + c.Assert(leader.GetServer().GetServiceMiddlewarePersistOptions().GetRateLimitConfig().LimiterConfig, HasLen, 1) } func mustWaitLeader(c *C, svrs map[string]*tests.TestServer) *server.Server { From c637fb0b3847a8598b3064384ab981fe51d41a5f Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 18 May 2022 17:47:56 +0800 Subject: [PATCH 20/56] merge master Signed-off-by: Cabinfever_B --- server/api/util.go | 153 --------------------------------------------- 1 file changed, 153 deletions(-) delete mode 100644 server/api/util.go diff --git a/server/api/util.go b/server/api/util.go deleted file mode 100644 index e9b18e6e85e..00000000000 --- a/server/api/util.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2016 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "bytes" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - - "github.com/pingcap/errors" - "github.com/tikv/pd/pkg/apiutil" -) - -var ( - errNoImplement = errors.New("no implement") - errOptionNotExist = func(name string) error { return errors.Errorf("the option %s does not exist", name) } -) - -func collectEscapeStringOption(option string, input map[string]interface{}, collectors ...func(v string)) error { - if v, ok := input[option].(string); ok { - value, err := url.QueryUnescape(v) - if err != nil { - return err - } - for _, c := range collectors { - c(value) - } - return nil - } - return errOptionNotExist(option) -} - -func collectStringOption(option string, input map[string]interface{}, collectors ...func(v string)) error { - if v, ok := input[option].(string); ok { - for _, c := range collectors { - c(v) - } - return nil - } - return errOptionNotExist(option) -} - -func readJSON(client *http.Client, url string, data interface{}) error { - resp, err := client.Get(url) - if err != nil { - return err - } - return extractJSON(resp, data) -} - -func readJSONWithBody(client *http.Client, url string, body []byte, data interface{}) error { - req, err := http.NewRequest(http.MethodGet, url, bytes.NewBuffer(body)) - if err != nil { - return err - } - resp, err := client.Do(req) - if err != nil { - return err - } - return extractJSON(resp, data) -} - -func extractJSON(resp *http.Response, data interface{}) error { - b, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK { - return errors.Errorf("http get url %s return code %d", resp.Request.URL, resp.StatusCode) - } - err = json.Unmarshal(b, data) - if err != nil { - return errors.WithStack(err) - } - return nil -} - -func postJSON(client *http.Client, url string, data []byte, checkOpts ...func([]byte, int)) error { - return apiutil.PostJSON(client, url, data, checkOpts...) -} - -func postJSONIgnoreRespStatus(client *http.Client, url string, data []byte, checkOpts ...func([]byte, int)) error { - return apiutil.PostJSONIgnoreRespStatus(client, url, data, checkOpts...) -} - -func getJSON(client *http.Client, url string, data []byte, checkOpts ...func([]byte, int)) error { - return apiutil.GetJSON(client, url, data, checkOpts...) -} - -func patchJSON(client *http.Client, url string, body []byte) error { - req, err := http.NewRequest(http.MethodPatch, url, bytes.NewBuffer(body)) - if err != nil { - return err - } - resp, err := client.Do(req) - if err != nil { - return errors.WithStack(err) - } - defer resp.Body.Close() - res, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK { - return errors.New(string(res)) - } - return nil -} - -func doDelete(client *http.Client, url string) (int, error) { - req, err := http.NewRequest(http.MethodDelete, url, nil) - if err != nil { - return http.StatusBadRequest, err - } - res, err := client.Do(req) - if err != nil { - return 0, err - } - defer res.Body.Close() - return res.StatusCode, nil -} - -func parseKey(name string, input map[string]interface{}) ([]byte, string, error) { - k, ok := input[name] - if !ok { - return nil, "", fmt.Errorf("missing %s", name) - } - rawKey, ok := k.(string) - if !ok { - return nil, "", fmt.Errorf("bad format %s", name) - } - returned, err := hex.DecodeString(rawKey) - if err != nil { - return nil, "", fmt.Errorf("split key %s is not in hex format", name) - } - return returned, rawKey, nil -} From 273a97078271fc640f6b43d096602423e96a133c Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 18 May 2022 18:17:55 +0800 Subject: [PATCH 21/56] merge master Signed-off-by: Cabinfever_B --- pkg/apiutil/apiutil.go | 10 --- server/api/router.go | 2 +- server/api/server_test.go | 2 +- server/api/service_middleware.go | 18 ++++- server/api/service_middleware_test.go | 103 ++++++++------------------ 5 files changed, 46 insertions(+), 89 deletions(-) diff --git a/pkg/apiutil/apiutil.go b/pkg/apiutil/apiutil.go index 56d2a0a0f81..1989df33b2c 100644 --- a/pkg/apiutil/apiutil.go +++ b/pkg/apiutil/apiutil.go @@ -171,16 +171,6 @@ func PostJSON(client *http.Client, url string, data []byte) (*http.Response, err return client.Do(req) } -// PostJSONIgnoreRespStatus is used to send the POST request to a specific URL and ignore resp status to test -func PostJSONIgnoreRespStatus(client *http.Client, url string, data []byte, checkOpts ...func([]byte, int)) error { - req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - return doJSONIgnoreRespStatus(client, req, checkOpts...) -} - // GetJSON is used to send GET requst to specific url func GetJSON(client *http.Client, url string, data []byte) (*http.Response, error) { req, err := http.NewRequest(http.MethodGet, url, bytes.NewBuffer(data)) diff --git a/server/api/router.go b/server/api/router.go index 8fa2f7573df..b92d2a6cf07 100644 --- a/server/api/router.go +++ b/server/api/router.go @@ -281,11 +281,11 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(clusterRouter, "/admin/reset-ts", adminHandler.ResetTS, setMethods("POST"), setAuditBackend(localLog)) registerFunc(apiRouter, "/admin/persist-file/{file_name}", adminHandler.SavePersistFile, setMethods("POST"), setAuditBackend(localLog)) registerFunc(clusterRouter, "/admin/replication_mode/wait-async", adminHandler.UpdateWaitAsyncTime, setMethods("POST"), setAuditBackend(localLog)) - registerFunc(apiRouter, "/admin/ratelimit/config", adminHandler.SetRatelimitConfig, setMethods("POST"), setAuditBackend(localLog)) serviceMiddlewareHandler := newServiceMiddlewareHandler(svr, rd) registerFunc(apiRouter, "/service-middleware/config", serviceMiddlewareHandler.GetServiceMiddlewareConfig, setMethods("GET")) registerFunc(apiRouter, "/service-middleware/config", serviceMiddlewareHandler.SetServiceMiddlewareConfig, setMethods("POST"), setAuditBackend(localLog)) + registerFunc(apiRouter, "/service-middleware/rate-limit/config", serviceMiddlewareHandler.SetRatelimitConfig, setMethods("POST"), setAuditBackend(localLog)) logHandler := newLogHandler(svr, rd) registerFunc(apiRouter, "/admin/log", logHandler.SetLogLevel, setMethods("POST"), setAuditBackend(localLog)) diff --git a/server/api/server_test.go b/server/api/server_test.go index 51467db3938..2b835987870 100644 --- a/server/api/server_test.go +++ b/server/api/server_test.go @@ -173,7 +173,7 @@ func (s *testServerServiceSuite) TearDownSuite(c *C) { s.cleanup() } -func (s *testServiceSuite) TestServiceLabels(c *C) { +func (s *testServerServiceSuite) TestServiceLabels(c *C) { accessPaths := s.svr.GetServiceLabels("Profile") c.Assert(accessPaths, HasLen, 1) c.Assert(accessPaths[0].Path, Equals, "/pd/api/v1/debug/pprof/profile") diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 4acca64ddb6..9a4cafb8dfc 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -156,14 +156,24 @@ func (h *serviceMiddlewareHandler) updateRateLimit(config *config.ServiceMiddlew return err } +func (h *serviceMiddlewareHandler) updateRateLimitConfig(key, label string, value ratelimit.DimensionConfig) error { + cfg := h.svr.GetServiceMiddlewareConfig() + rateLimitCfg := ratelimit.NewLimiterConfig() + for label, item := range cfg.LimiterConfig { + rateLimitCfg[label] = item + } + rateLimitCfg[label] = value + return h.updateRateLimit(cfg, key, &rateLimitCfg) +} + // @Tags service_middleware // @Summary update ratelimit config // @Param body body object string "json params" // @Produce json // @Success 200 {string} string "" // @Failure 400 {string} string "" -// @Router /service-middleware/config [POST] -func (h *adminHandler) SetRatelimitConfig(w http.ResponseWriter, r *http.Request) { +// @Router /service-middleware/rate-limit/config [POST] +func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r *http.Request) { var input map[string]interface{} if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &input); err != nil { return @@ -226,7 +236,7 @@ func (h *adminHandler) SetRatelimitConfig(w http.ResponseWriter, r *http.Request } } cfg.QPS = qps - cfg.QPSBrust = brust + cfg.QPSBurst = brust } status := h.svr.UpdateServiceRateLimiter(serviceLabel, ratelimit.UpdateDimensionConfig(cfg)) switch { @@ -244,7 +254,7 @@ func (h *adminHandler) SetRatelimitConfig(w http.ResponseWriter, r *http.Request if !okc && !okq { h.rd.JSON(w, http.StatusOK, "No changed.") } else { - err := updateRateLimitConfig(h.svr, "rate-limit-config", serviceLabel, cfg) + err := h.updateRateLimitConfig("limiter-config", serviceLabel, cfg) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) } else { diff --git a/server/api/service_middleware_test.go b/server/api/service_middleware_test.go index e956308bb9f..02a66323563 100644 --- a/server/api/service_middleware_test.go +++ b/server/api/service_middleware_test.go @@ -18,7 +18,6 @@ import ( "encoding/json" "fmt" "net/http" - "strings" . "github.com/pingcap/check" "github.com/pingcap/failpoint" @@ -126,7 +125,7 @@ func (s *testRateLimitConfigSuite) TearDownSuite(c *C) { } func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { - urlPrefix := fmt.Sprintf("%s%s/api/v1/admin/ratelimit/config", s.svr.GetAddr(), apiPrefix) + urlPrefix := fmt.Sprintf("%s%s/api/v1/service-middleware/config", s.svr.GetAddr(), apiPrefix) // test empty type input := make(map[string]interface{}) @@ -134,22 +133,16 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { jsonBody, err := json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(string(res), Equals, "\"The type is empty.\"\n") - c.Assert(code, Equals, http.StatusBadRequest) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.Status(c, http.StatusBadRequest), tu.StringEqual(c, "\"The type is empty.\"\n")) c.Assert(err, IsNil) // test invalid type input = make(map[string]interface{}) input["type"] = "url" jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(string(res), Equals, "\"The type is invalid.\"\n") - c.Assert(code, Equals, http.StatusBadRequest) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.Status(c, http.StatusBadRequest), tu.StringEqual(c, "\"The type is invalid.\"\n")) c.Assert(err, IsNil) // test empty label @@ -158,11 +151,8 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { input["label"] = "" jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(string(res), Equals, "\"The label is empty.\"\n") - c.Assert(code, Equals, http.StatusBadRequest) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.Status(c, http.StatusBadRequest), tu.StringEqual(c, "\"The label is empty.\"\n")) c.Assert(err, IsNil) // test no label matched input = make(map[string]interface{}) @@ -170,11 +160,8 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { input["label"] = "TestLabel" jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(string(res), Equals, "\"There is no label matched.\"\n") - c.Assert(code, Equals, http.StatusBadRequest) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.Status(c, http.StatusBadRequest), tu.StringEqual(c, "\"There is no label matched.\"\n")) c.Assert(err, IsNil) // test empty path @@ -183,11 +170,8 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { input["path"] = "" jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(string(res), Equals, "\"The path is empty.\"\n") - c.Assert(code, Equals, http.StatusBadRequest) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.Status(c, http.StatusBadRequest), tu.StringEqual(c, "\"The path is empty.\"\n")) c.Assert(err, IsNil) // test path but no label matched @@ -196,11 +180,8 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { input["path"] = "/pd/api/v1/test" jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(string(res), Equals, "\"There is no label matched.\"\n") - c.Assert(code, Equals, http.StatusBadRequest) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.Status(c, http.StatusBadRequest), tu.StringEqual(c, "\"There is no label matched.\"\n")) c.Assert(err, IsNil) // no change @@ -209,11 +190,8 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { input["label"] = "GetHealthStatus" jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(string(res), Equals, "\"No changed.\"\n") - c.Assert(code, Equals, http.StatusOK) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.StatusOK(c), tu.StringEqual(c, "\"No changed.\"\n")) c.Assert(err, IsNil) // change concurrency @@ -224,20 +202,14 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { input["concurrency"] = 100 jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(strings.Contains(string(res), "Concurrency limiter is changed."), Equals, true) - c.Assert(code, Equals, http.StatusOK) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.StatusOK(c), tu.StringEqual(c, "Concurrency limiter is changed.")) c.Assert(err, IsNil) input["concurrency"] = 0 jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(strings.Contains(string(res), "Concurrency limiter is deleted."), Equals, true) - c.Assert(code, Equals, http.StatusOK) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.StatusOK(c), tu.StringEqual(c, "Concurrency limiter is deleted.")) c.Assert(err, IsNil) // change qps @@ -248,11 +220,8 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { input["qps"] = 100 jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(strings.Contains(string(res), "QPS rate limiter is changed."), Equals, true) - c.Assert(code, Equals, http.StatusOK) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.StatusOK(c), tu.StringEqual(c, "QPS rate limiter is changed.")) c.Assert(err, IsNil) input = make(map[string]interface{}) @@ -262,22 +231,16 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { input["qps"] = 0.3 jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(strings.Contains(string(res), "QPS rate limiter is changed."), Equals, true) - c.Assert(code, Equals, http.StatusOK) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.StatusOK(c), tu.StringEqual(c, "QPS rate limiter is changed.")) c.Assert(err, IsNil) - c.Assert(s.svr.GetConfig().PDServerCfg.RateLimitConfig["GetHealthStatus"].QPSBrust, Equals, 1) + c.Assert(s.svr.GetRateLimitConfig().LimiterConfig["GetHealthStatus"].QPSBurst, Equals, 1) input["qps"] = -1 jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(strings.Contains(string(res), "QPS rate limiter is deleted."), Equals, true) - c.Assert(code, Equals, http.StatusOK) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.StatusOK(c), tu.StringEqual(c, "QPS rate limiter is deleted.")) c.Assert(err, IsNil) // change both @@ -288,11 +251,8 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { input["concurrency"] = 100 jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(string(res), Equals, "\"Concurrency limiter is changed. QPS rate limiter is changed.\"\n") - c.Assert(code, Equals, http.StatusOK) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.StatusOK(c), tu.StringEqual(c, "\"Concurrency limiter is changed. QPS rate limiter is changed.\"\n")) c.Assert(err, IsNil) limiter := s.svr.GetServiceRateLimiter() @@ -306,10 +266,7 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { input["concurrency"] = 100 jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) - err = postJSONIgnoreRespStatus(testDialClient, urlPrefix, jsonBody, - func(res []byte, code int) { - c.Assert(string(res), Equals, "\"This service is in block list.\"\n") - c.Assert(code, Equals, http.StatusBadRequest) - }) + err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, + tu.StatusOK(c), tu.StringEqual(c, "\"This service is in block list.\"\n")) c.Assert(err, IsNil) } From c87f49bdf09819e40bb6738a06f3590c947c139b Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 18 May 2022 18:33:14 +0800 Subject: [PATCH 22/56] fix test Signed-off-by: Cabinfever_B --- server/api/service_middleware_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/api/service_middleware_test.go b/server/api/service_middleware_test.go index 02a66323563..7df3b2c8d78 100644 --- a/server/api/service_middleware_test.go +++ b/server/api/service_middleware_test.go @@ -125,7 +125,7 @@ func (s *testRateLimitConfigSuite) TearDownSuite(c *C) { } func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { - urlPrefix := fmt.Sprintf("%s%s/api/v1/service-middleware/config", s.svr.GetAddr(), apiPrefix) + urlPrefix := fmt.Sprintf("%s%s/api/v1/service-middleware/rate-limit/config", s.svr.GetAddr(), apiPrefix) // test empty type input := make(map[string]interface{}) @@ -267,6 +267,6 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, - tu.StatusOK(c), tu.StringEqual(c, "\"This service is in block list.\"\n")) + tu.StatusNotOK(c), tu.StringEqual(c, "\"This service is in block list.\"\n")) c.Assert(err, IsNil) } From b29445d5a375cea882fdc58eca547b2cad78b840 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 18 May 2022 18:48:41 +0800 Subject: [PATCH 23/56] allow list Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 4 ++-- server/api/service_middleware_test.go | 2 +- server/server.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 9a4cafb8dfc..505112133dc 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -211,8 +211,8 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * h.rd.JSON(w, http.StatusBadRequest, "The type is invalid.") return } - if h.svr.IsInRateLimitBlockList(serviceLabel) { - h.rd.JSON(w, http.StatusBadRequest, "This service is in block list.") + if h.svr.IsInRateLimitAllowList(serviceLabel) { + h.rd.JSON(w, http.StatusBadRequest, "This service is in allow list whose config can not be changed.") return } cfg := h.svr.GetRateLimitConfig().LimiterConfig[serviceLabel] diff --git a/server/api/service_middleware_test.go b/server/api/service_middleware_test.go index 7df3b2c8d78..a1b60a25137 100644 --- a/server/api/service_middleware_test.go +++ b/server/api/service_middleware_test.go @@ -267,6 +267,6 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, - tu.StatusNotOK(c), tu.StringEqual(c, "\"This service is in block list.\"\n")) + tu.StatusNotOK(c), tu.StringEqual(c, "\"This service is in allow list whose config can not be changed.\"\n")) c.Assert(err, IsNil) } diff --git a/server/server.go b/server/server.go index f0e3c078f99..297ec29a11f 100644 --- a/server/server.go +++ b/server/server.go @@ -1234,8 +1234,8 @@ func (s *Server) GetServiceRateLimiter() *ratelimit.Limiter { return s.serviceRateLimiter } -// IsInRateLimitBlockList returns whethis given service label is in block lost -func (s *Server) IsInRateLimitBlockList(serviceLabel string) bool { +// IsInRateLimitAllowList returns whethis given service label is in allow lost +func (s *Server) IsInRateLimitAllowList(serviceLabel string) bool { return s.serviceRateLimiter.IsInAllowList(serviceLabel) } From f18956efe3231238b53a4a4e35032bef3893dd0c Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Thu, 19 May 2022 17:34:02 +0800 Subject: [PATCH 24/56] fix Signed-off-by: Cabinfever_B --- pkg/ratelimit/limiter.go | 7 +------ pkg/ratelimit/option.go | 4 +--- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/pkg/ratelimit/limiter.go b/pkg/ratelimit/limiter.go index 45661f38216..b9c9155b5e9 100644 --- a/pkg/ratelimit/limiter.go +++ b/pkg/ratelimit/limiter.go @@ -23,11 +23,6 @@ import ( // LimiterConfig is the config of Limiter type LimiterConfig map[string]DimensionConfig -// NewLimiterConfig returns a new LimiterConfig -func NewLimiterConfig() LimiterConfig { - return make(map[string]DimensionConfig) -} - // DimensionConfig is the limit dimension config of one label type DimensionConfig struct { // qps conifg @@ -51,7 +46,7 @@ type Limiter struct { func NewLimiter() *Limiter { return &Limiter{ labelAllowList: make(map[string]struct{}), - labelConfig: NewLimiterConfig(), + labelConfig: make(map[string]DimensionConfig), } } diff --git a/pkg/ratelimit/option.go b/pkg/ratelimit/option.go index 79e07fa736b..e4d5b88ecf1 100644 --- a/pkg/ratelimit/option.go +++ b/pkg/ratelimit/option.go @@ -35,8 +35,6 @@ const ( ConcurrencyDeleted // InAllowList shows that limiter's config isn't changed because it is in in allow list. InAllowList - // Ignore shows that the status can be ignored when initiated - Ignore ) // Option is used to create a limiter with the optional settings. @@ -48,7 +46,7 @@ type Option func(string, *Limiter) UpdateStatus func AddLabelAllowList() Option { return func(label string, l *Limiter) UpdateStatus { l.labelAllowList[label] = struct{}{} - return Ignore + return 0 } } From 6e0819c23741595cf902925636c452489002df7b Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Thu, 19 May 2022 18:01:15 +0800 Subject: [PATCH 25/56] fix Signed-off-by: Cabinfever_B --- pkg/ratelimit/limiter.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/ratelimit/limiter.go b/pkg/ratelimit/limiter.go index b9c9155b5e9..ee150ec61f4 100644 --- a/pkg/ratelimit/limiter.go +++ b/pkg/ratelimit/limiter.go @@ -20,9 +20,6 @@ import ( "golang.org/x/time/rate" ) -// LimiterConfig is the config of Limiter -type LimiterConfig map[string]DimensionConfig - // DimensionConfig is the limit dimension config of one label type DimensionConfig struct { // qps conifg @@ -35,7 +32,7 @@ type DimensionConfig struct { // Limiter is a controller for the request rate. type Limiter struct { configMux sync.Mutex - labelConfig LimiterConfig + labelConfig map[string]DimensionConfig qpsLimiter sync.Map concurrencyLimiter sync.Map // the label which is in labelAllowList won't be limited From 9988a7ee4cd9f1251deb2381ce11ae0df6219263 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 25 May 2022 14:53:29 +0800 Subject: [PATCH 26/56] change into point Signed-off-by: Cabinfever_B --- pkg/ratelimit/limiter.go | 8 ++++---- pkg/ratelimit/option.go | 12 ++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/ratelimit/limiter.go b/pkg/ratelimit/limiter.go index ee150ec61f4..34655f900e5 100644 --- a/pkg/ratelimit/limiter.go +++ b/pkg/ratelimit/limiter.go @@ -32,7 +32,7 @@ type DimensionConfig struct { // Limiter is a controller for the request rate. type Limiter struct { configMux sync.Mutex - labelConfig map[string]DimensionConfig + labelConfig map[string]*DimensionConfig qpsLimiter sync.Map concurrencyLimiter sync.Map // the label which is in labelAllowList won't be limited @@ -43,7 +43,7 @@ type Limiter struct { func NewLimiter() *Limiter { return &Limiter{ labelAllowList: make(map[string]struct{}), - labelConfig: make(map[string]DimensionConfig), + labelConfig: make(map[string]*DimensionConfig), } } @@ -96,7 +96,7 @@ func (l *Limiter) GetQPSLimiterStatus(label string) (limit rate.Limit, burst int return 0, 0 } -// deleteQPSLimiter deletes QPS limiter of a given label +// deleteQPSLimiter deletes QPS limiter of the given label func (l *Limiter) deleteQPSLimiter(label string) { l.qpsLimiter.Delete(label) } @@ -110,7 +110,7 @@ func (l *Limiter) GetConcurrencyLimiterStatus(label string) (limit uint64, curre return 0, 0 } -// deleteConcurrencyLimiter deletes concurrency limiter of a given label +// deleteConcurrencyLimiter deletes concurrency limiter of the given label func (l *Limiter) deleteConcurrencyLimiter(label string) { l.concurrencyLimiter.Delete(label) } diff --git a/pkg/ratelimit/option.go b/pkg/ratelimit/option.go index e4d5b88ecf1..226097892c3 100644 --- a/pkg/ratelimit/option.go +++ b/pkg/ratelimit/option.go @@ -54,7 +54,11 @@ func updateConcurrencyConfig(l *Limiter, label string, limit uint64) UpdateStatu l.configMux.Lock() defer l.configMux.Unlock() - cfg := l.labelConfig[label] + cfg, ok := l.labelConfig[label] + if !ok { + cfg = &DimensionConfig{} + l.labelConfig[label] = cfg + } if cfg.ConcurrencyLimit == limit { return ConcurrencyNoChange } @@ -74,7 +78,11 @@ func updateQPSConfig(l *Limiter, label string, limit float64, burst int) UpdateS l.configMux.Lock() defer l.configMux.Unlock() - cfg := l.labelConfig[label] + cfg, ok := l.labelConfig[label] + if !ok { + cfg = &DimensionConfig{} + l.labelConfig[label] = cfg + } if cfg.QPS == limit && cfg.QPSBurst == burst { return QPSNoChange } From e6824076c1d933b7471d33cf6be79b992186520d Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 25 May 2022 15:43:01 +0800 Subject: [PATCH 27/56] change into point Signed-off-by: Cabinfever_B --- pkg/ratelimit/limiter_test.go | 2 +- pkg/ratelimit/option.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ratelimit/limiter_test.go b/pkg/ratelimit/limiter_test.go index f1ae1662fa4..cf75d76152a 100644 --- a/pkg/ratelimit/limiter_test.go +++ b/pkg/ratelimit/limiter_test.go @@ -194,7 +194,7 @@ func (s *testRatelimiterSuite) TestQPSLimiter(c *C) { func (s *testRatelimiterSuite) TestTwoLimiters(c *C) { c.Parallel() - cfg := DimensionConfig{ + cfg := &DimensionConfig{ QPS: 100, QPSBurst: 100, ConcurrencyLimit: 100, diff --git a/pkg/ratelimit/option.go b/pkg/ratelimit/option.go index 226097892c3..90c6c2ce4fe 100644 --- a/pkg/ratelimit/option.go +++ b/pkg/ratelimit/option.go @@ -121,7 +121,7 @@ func UpdateQPSLimiter(limit float64, burst int) Option { } // UpdateDimensionConfig creates QPS limiter and concurrency limiter for a given label by config if it doesn't exist. -func UpdateDimensionConfig(cfg DimensionConfig) Option { +func UpdateDimensionConfig(cfg *DimensionConfig) Option { return func(label string, l *Limiter) UpdateStatus { if _, allow := l.labelAllowList[label]; allow { return InAllowList From 24fa55be3cd5d8804583c6145eb818da27a299ee Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 25 May 2022 15:55:38 +0800 Subject: [PATCH 28/56] merge Signed-off-by: Cabinfever_B --- go.mod | 1 + go.sum | 1 + server/config/service_middleware_config.go | 2 +- tests/server/config/config_test.go | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 875eb6efe7c..aab95d5d965 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/AlekSi/gocov-xml v1.0.0 github.com/BurntSushi/toml v0.3.1 + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/aws/aws-sdk-go v1.35.3 github.com/axw/gocov v1.0.0 github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 diff --git a/go.sum b/go.sum index 50997691e2a..702a4b7866c 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,7 @@ github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQ github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502 h1:L8IbaI/W6h5Cwgh0n4zGeZpVK78r/jBf9ASurHo9+/o= github.com/Xeoncross/go-aesctr-with-hmac v0.0.0-20200623134604-12b17a7ff502/go.mod h1:pmnBM9bxWSiHvC/gSWunUIyDvGn33EkP2CUjxFKtTTM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= diff --git a/server/config/service_middleware_config.go b/server/config/service_middleware_config.go index 15dc8e1fd38..446782c37c0 100644 --- a/server/config/service_middleware_config.go +++ b/server/config/service_middleware_config.go @@ -65,7 +65,7 @@ type RateLimitConfig struct { // EnableRateLimit controls the switch of the rate limit middleware EnableRateLimit bool `json:"enable-rate-limit,string"` // RateLimitConfig is the config of rate limit middleware - LimiterConfig ratelimit.LimiterConfig `json:"limiter-config"` + LimiterConfig map[string]*ratelimit.DimensionConfig `json:"limiter-config"` } // Clone returns a cloned rate limit config. diff --git a/tests/server/config/config_test.go b/tests/server/config/config_test.go index 7ce084243e3..811bf9e3254 100644 --- a/tests/server/config/config_test.go +++ b/tests/server/config/config_test.go @@ -66,8 +66,8 @@ func (s *testConfigPresistSuite) TestRateLimitConfigReload(c *C) { leader := s.cluster.GetServer(s.cluster.GetLeader()) c.Assert(leader.GetServer().GetServiceMiddlewareConfig().RateLimitConfig.LimiterConfig, HasLen, 0) - limitCfg := ratelimit.NewLimiterConfig() - limitCfg["GetRegions"] = ratelimit.DimensionConfig{QPS: 1} + limitCfg := make(map[string]*ratelimit.DimensionConfig) + limitCfg["GetRegions"] = &ratelimit.DimensionConfig{QPS: 1} input := map[string]interface{}{ "enable-rate-limit": "true", From 3d87b640044d2c8f2e88a8841a30390f2912f9e4 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 25 May 2022 15:56:32 +0800 Subject: [PATCH 29/56] merge Signed-off-by: Cabinfever_B --- server/server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/server.go b/server/server.go index 297ec29a11f..3a892665eea 100644 --- a/server/server.go +++ b/server/server.go @@ -262,7 +262,6 @@ func CreateServer(ctx context.Context, cfg *config.Config, serviceBuilders ...Ha } s.serviceRateLimiter = ratelimit.NewLimiter() s.serviceAuditBackendLabels = make(map[string]*audit.BackendLabels) - s.serviceRateLimiter = ratelimit.NewLimiter() s.serviceLabels = make(map[string][]apiutil.AccessPath) s.apiServiceLabelMap = make(map[apiutil.AccessPath]string) From 4f93cac36ec43c92b5842570506084ebc8974424 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 25 May 2022 16:54:25 +0800 Subject: [PATCH 30/56] merge Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 505112133dc..6615866ea16 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -156,9 +156,9 @@ func (h *serviceMiddlewareHandler) updateRateLimit(config *config.ServiceMiddlew return err } -func (h *serviceMiddlewareHandler) updateRateLimitConfig(key, label string, value ratelimit.DimensionConfig) error { +func (h *serviceMiddlewareHandler) updateRateLimitConfig(key, label string, value *ratelimit.DimensionConfig) error { cfg := h.svr.GetServiceMiddlewareConfig() - rateLimitCfg := ratelimit.NewLimiterConfig() + rateLimitCfg := make(map[string]*ratelimit.DimensionConfig) for label, item := range cfg.LimiterConfig { rateLimitCfg[label] = item } From 90227242d0b239fa1129eb2e8093c9b276ae3499 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 25 May 2022 17:28:16 +0800 Subject: [PATCH 31/56] merge Signed-off-by: Cabinfever_B --- server/config/service_middleware_config.go | 2 +- tests/server/config/config_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/config/service_middleware_config.go b/server/config/service_middleware_config.go index 15dc8e1fd38..9cce425fb92 100644 --- a/server/config/service_middleware_config.go +++ b/server/config/service_middleware_config.go @@ -65,7 +65,7 @@ type RateLimitConfig struct { // EnableRateLimit controls the switch of the rate limit middleware EnableRateLimit bool `json:"enable-rate-limit,string"` // RateLimitConfig is the config of rate limit middleware - LimiterConfig ratelimit.LimiterConfig `json:"limiter-config"` + LimiterConfig map[string]ratelimit.DimensionConfig `json:"limiter-config"` } // Clone returns a cloned rate limit config. diff --git a/tests/server/config/config_test.go b/tests/server/config/config_test.go index 7ce084243e3..f80dd8ceb38 100644 --- a/tests/server/config/config_test.go +++ b/tests/server/config/config_test.go @@ -66,7 +66,7 @@ func (s *testConfigPresistSuite) TestRateLimitConfigReload(c *C) { leader := s.cluster.GetServer(s.cluster.GetLeader()) c.Assert(leader.GetServer().GetServiceMiddlewareConfig().RateLimitConfig.LimiterConfig, HasLen, 0) - limitCfg := ratelimit.NewLimiterConfig() + limitCfg := make(map[string]ratelimit.DimensionConfig) limitCfg["GetRegions"] = ratelimit.DimensionConfig{QPS: 1} input := map[string]interface{}{ From db80b3cfbc172510015e3e25911eacefab2e72de Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 25 May 2022 17:29:02 +0800 Subject: [PATCH 32/56] merge Signed-off-by: Cabinfever_B --- server/server.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/server.go b/server/server.go index 246d55bd5ea..da8379eb23b 100644 --- a/server/server.go +++ b/server/server.go @@ -1508,8 +1508,9 @@ func (s *Server) reloadConfigFromKV() error { func (s *Server) loadRateLimitConfig() { cfg := s.serviceMiddlewarePersistOptions.GetRateLimitConfig().LimiterConfig - for key, value := range cfg { - s.serviceRateLimiter.Update(key, ratelimit.UpdateDimensionConfig(value)) + for key := range cfg { + value := cfg[key] + s.serviceRateLimiter.Update(key, ratelimit.UpdateDimensionConfig(&value)) } } From 76f347cf318a92991429bff72d95e654013d62ad Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 25 May 2022 18:13:27 +0800 Subject: [PATCH 33/56] merge Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 6615866ea16..2dc254f2eb8 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -156,9 +156,9 @@ func (h *serviceMiddlewareHandler) updateRateLimit(config *config.ServiceMiddlew return err } -func (h *serviceMiddlewareHandler) updateRateLimitConfig(key, label string, value *ratelimit.DimensionConfig) error { +func (h *serviceMiddlewareHandler) updateRateLimitConfig(key, label string, value ratelimit.DimensionConfig) error { cfg := h.svr.GetServiceMiddlewareConfig() - rateLimitCfg := make(map[string]*ratelimit.DimensionConfig) + rateLimitCfg := make(map[string]ratelimit.DimensionConfig) for label, item := range cfg.LimiterConfig { rateLimitCfg[label] = item } @@ -238,7 +238,7 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * cfg.QPS = qps cfg.QPSBurst = brust } - status := h.svr.UpdateServiceRateLimiter(serviceLabel, ratelimit.UpdateDimensionConfig(cfg)) + status := h.svr.UpdateServiceRateLimiter(serviceLabel, ratelimit.UpdateDimensionConfig(&cfg)) switch { case status&ratelimit.QPSChanged != 0: qpsRateUpdatedFlag = "QPS rate limiter is changed." From f6407628d8b75a9b05924324d3f74e28ad064aa6 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Thu, 26 May 2022 17:54:19 +0800 Subject: [PATCH 34/56] remove unnecessary code Signed-off-by: Cabinfever_B --- pkg/ratelimit/limiter.go | 8 ++++---- pkg/ratelimit/option.go | 9 ++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/ratelimit/limiter.go b/pkg/ratelimit/limiter.go index 34655f900e5..38702e921e3 100644 --- a/pkg/ratelimit/limiter.go +++ b/pkg/ratelimit/limiter.go @@ -96,8 +96,8 @@ func (l *Limiter) GetQPSLimiterStatus(label string) (limit rate.Limit, burst int return 0, 0 } -// deleteQPSLimiter deletes QPS limiter of the given label -func (l *Limiter) deleteQPSLimiter(label string) { +// QPSUnlimit deletes QPS limiter of the given label +func (l *Limiter) QPSUnlimit(label string) { l.qpsLimiter.Delete(label) } @@ -110,8 +110,8 @@ func (l *Limiter) GetConcurrencyLimiterStatus(label string) (limit uint64, curre return 0, 0 } -// deleteConcurrencyLimiter deletes concurrency limiter of the given label -func (l *Limiter) deleteConcurrencyLimiter(label string) { +// ConcurrencyUnlimit deletes concurrency limiter of the given label +func (l *Limiter) ConcurrencyUnlimit(label string) { l.concurrencyLimiter.Delete(label) } diff --git a/pkg/ratelimit/option.go b/pkg/ratelimit/option.go index 90c6c2ce4fe..885a3646f79 100644 --- a/pkg/ratelimit/option.go +++ b/pkg/ratelimit/option.go @@ -21,6 +21,7 @@ type UpdateStatus uint32 // Flags for limiter. const ( + eps float64 = 1e-8 // QPSNoChange shows that limiter's config isn't changed. QPSNoChange UpdateStatus = 1 << iota // QPSChanged shows that limiter's config is changed and not deleted. @@ -63,9 +64,8 @@ func updateConcurrencyConfig(l *Limiter, label string, limit uint64) UpdateStatu return ConcurrencyNoChange } cfg.ConcurrencyLimit = limit - l.labelConfig[label] = cfg if limit < 1 { - l.deleteConcurrencyLimiter(label) + l.ConcurrencyUnlimit(label) return ConcurrencyDeleted } if limiter, exist := l.concurrencyLimiter.LoadOrStore(label, newConcurrencyLimiter(limit)); exist { @@ -83,14 +83,13 @@ func updateQPSConfig(l *Limiter, label string, limit float64, burst int) UpdateS cfg = &DimensionConfig{} l.labelConfig[label] = cfg } - if cfg.QPS == limit && cfg.QPSBurst == burst { + if (cfg.QPS-limit < eps && cfg.QPS-limit > -eps) && cfg.QPSBurst == burst { return QPSNoChange } cfg.QPS = limit cfg.QPSBurst = burst - l.labelConfig[label] = cfg if limit <= 0 || burst < 1 { - l.deleteQPSLimiter(label) + l.QPSUnlimit(label) return QPSDeleted } if limiter, exist := l.qpsLimiter.LoadOrStore(label, NewRateLimiter(limit, burst)); exist { From 099e49a904b2e41752d8a86b3fe0e36eba0797ec Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Thu, 26 May 2022 17:56:52 +0800 Subject: [PATCH 35/56] fix type Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 2dc254f2eb8..a737d3e05ac 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -238,22 +238,22 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * cfg.QPS = qps cfg.QPSBurst = brust } - status := h.svr.UpdateServiceRateLimiter(serviceLabel, ratelimit.UpdateDimensionConfig(&cfg)) - switch { - case status&ratelimit.QPSChanged != 0: - qpsRateUpdatedFlag = "QPS rate limiter is changed." - case status&ratelimit.QPSDeleted != 0: - qpsRateUpdatedFlag = "QPS rate limiter is deleted." - } - switch { - case status&ratelimit.ConcurrencyChanged != 0: - concurrencyUpdatedFlag = "Concurrency limiter is changed." - case status&ratelimit.ConcurrencyDeleted != 0: - concurrencyUpdatedFlag = "Concurrency limiter is deleted." - } if !okc && !okq { h.rd.JSON(w, http.StatusOK, "No changed.") } else { + status := h.svr.UpdateServiceRateLimiter(serviceLabel, ratelimit.UpdateDimensionConfig(&cfg)) + switch { + case status&ratelimit.QPSChanged != 0: + qpsRateUpdatedFlag = "QPS rate limiter is changed." + case status&ratelimit.QPSDeleted != 0: + qpsRateUpdatedFlag = "QPS rate limiter is deleted." + } + switch { + case status&ratelimit.ConcurrencyChanged != 0: + concurrencyUpdatedFlag = "Concurrency limiter is changed." + case status&ratelimit.ConcurrencyDeleted != 0: + concurrencyUpdatedFlag = "Concurrency limiter is deleted." + } err := h.updateRateLimitConfig("limiter-config", serviceLabel, cfg) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) From 40832d895ff7eb55ecc41865627d7d90ce9c25c4 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 30 May 2022 10:34:51 +0800 Subject: [PATCH 36/56] remove config in Limiter Signed-off-by: Cabinfever_B --- pkg/ratelimit/limiter.go | 3 --- pkg/ratelimit/option.go | 24 ++++-------------------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/pkg/ratelimit/limiter.go b/pkg/ratelimit/limiter.go index 38702e921e3..4bf930ed6c5 100644 --- a/pkg/ratelimit/limiter.go +++ b/pkg/ratelimit/limiter.go @@ -31,8 +31,6 @@ type DimensionConfig struct { // Limiter is a controller for the request rate. type Limiter struct { - configMux sync.Mutex - labelConfig map[string]*DimensionConfig qpsLimiter sync.Map concurrencyLimiter sync.Map // the label which is in labelAllowList won't be limited @@ -43,7 +41,6 @@ type Limiter struct { func NewLimiter() *Limiter { return &Limiter{ labelAllowList: make(map[string]struct{}), - labelConfig: make(map[string]*DimensionConfig), } } diff --git a/pkg/ratelimit/option.go b/pkg/ratelimit/option.go index 885a3646f79..6dbdb482579 100644 --- a/pkg/ratelimit/option.go +++ b/pkg/ratelimit/option.go @@ -52,18 +52,10 @@ func AddLabelAllowList() Option { } func updateConcurrencyConfig(l *Limiter, label string, limit uint64) UpdateStatus { - l.configMux.Lock() - defer l.configMux.Unlock() - - cfg, ok := l.labelConfig[label] - if !ok { - cfg = &DimensionConfig{} - l.labelConfig[label] = cfg - } - if cfg.ConcurrencyLimit == limit { + oldConcurrencyLimit, _ := l.GetConcurrencyLimiterStatus(label) + if oldConcurrencyLimit == limit { return ConcurrencyNoChange } - cfg.ConcurrencyLimit = limit if limit < 1 { l.ConcurrencyUnlimit(label) return ConcurrencyDeleted @@ -75,19 +67,11 @@ func updateConcurrencyConfig(l *Limiter, label string, limit uint64) UpdateStatu } func updateQPSConfig(l *Limiter, label string, limit float64, burst int) UpdateStatus { - l.configMux.Lock() - defer l.configMux.Unlock() + oldQPSLimit, oldBurst := l.GetQPSLimiterStatus(label) - cfg, ok := l.labelConfig[label] - if !ok { - cfg = &DimensionConfig{} - l.labelConfig[label] = cfg - } - if (cfg.QPS-limit < eps && cfg.QPS-limit > -eps) && cfg.QPSBurst == burst { + if (float64(oldQPSLimit)-limit < eps && float64(oldQPSLimit)-limit > -eps) && oldBurst == burst { return QPSNoChange } - cfg.QPS = limit - cfg.QPSBurst = burst if limit <= 0 || burst < 1 { l.QPSUnlimit(label) return QPSDeleted From 5799d3774144108a691593823dcb17d65cf86f79 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 31 May 2022 17:09:05 +0800 Subject: [PATCH 37/56] fix eps Signed-off-by: Cabinfever_B --- pkg/ratelimit/option.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ratelimit/option.go b/pkg/ratelimit/option.go index 6dbdb482579..53afb9926d4 100644 --- a/pkg/ratelimit/option.go +++ b/pkg/ratelimit/option.go @@ -72,7 +72,7 @@ func updateQPSConfig(l *Limiter, label string, limit float64, burst int) UpdateS if (float64(oldQPSLimit)-limit < eps && float64(oldQPSLimit)-limit > -eps) && oldBurst == burst { return QPSNoChange } - if limit <= 0 || burst < 1 { + if limit <= eps || burst < 1 { l.QPSUnlimit(label) return QPSDeleted } From 5c55e8bf9d72cc7cac85520177537bd37d54d988 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 13 Jun 2022 15:37:56 +0800 Subject: [PATCH 38/56] add json merge util Signed-off-by: Cabinfever_B --- pkg/jsonutil/jsonutil.go | 47 ++++++++++++++++++++++++++++++++ server/api/config.go | 43 ++++------------------------- server/api/service_middleware.go | 8 ++---- 3 files changed, 54 insertions(+), 44 deletions(-) create mode 100644 pkg/jsonutil/jsonutil.go diff --git a/pkg/jsonutil/jsonutil.go b/pkg/jsonutil/jsonutil.go new file mode 100644 index 00000000000..d43643a73c2 --- /dev/null +++ b/pkg/jsonutil/jsonutil.go @@ -0,0 +1,47 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonutil + +import ( + "bytes" + "encoding/json" + + "github.com/tikv/pd/pkg/reflectutil" +) + +func AddKeyValue(old interface{}, key string, value interface{}) (updated bool, found bool, err error) { + data, err := json.Marshal(map[string]interface{}{key: value}) + if err != nil { + return false, false, err + } + return MergeJsonObject(old, data) +} + +func MergeJsonObject(v interface{}, data []byte) (updated bool, found bool, err error) { + old, _ := json.Marshal(v) + if err := json.Unmarshal(data, v); err != nil { + return false, false, err + } + new, _ := json.Marshal(v) + if !bytes.Equal(old, new) { + return true, true, nil + } + m := make(map[string]interface{}) + if err := json.Unmarshal(data, &m); err != nil { + return false, false, err + } + found = reflectutil.FindSameFieldByJSON(v, m) + return false, found, nil +} diff --git a/server/api/config.go b/server/api/config.go index de87947b785..724ff119198 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -15,7 +15,6 @@ package api import ( - "bytes" "encoding/json" "fmt" "io" @@ -29,6 +28,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/tikv/pd/pkg/apiutil" + "github.com/tikv/pd/pkg/jsonutil" "github.com/tikv/pd/pkg/logutil" "github.com/tikv/pd/pkg/reflectutil" "github.com/tikv/pd/server" @@ -164,12 +164,7 @@ func (h *confHandler) updateConfig(cfg *config.Config, key string, value interfa } func (h *confHandler) updateSchedule(config *config.Config, key string, value interface{}) error { - data, err := json.Marshal(map[string]interface{}{key: value}) - if err != nil { - return err - } - - updated, found, err := mergeConfig(&config.Schedule, data) + updated, found, err := jsonutil.AddKeyValue(&config.Schedule, key, value) if err != nil { return err } @@ -185,12 +180,7 @@ func (h *confHandler) updateSchedule(config *config.Config, key string, value in } func (h *confHandler) updateReplication(config *config.Config, key string, value interface{}) error { - data, err := json.Marshal(map[string]interface{}{key: value}) - if err != nil { - return err - } - - updated, found, err := mergeConfig(&config.Replication, data) + updated, found, err := jsonutil.AddKeyValue(&config.Replication, key, value) if err != nil { return err } @@ -212,8 +202,7 @@ func (h *confHandler) updateReplicationModeConfig(config *config.Config, key []s if err != nil { return err } - - updated, found, err := mergeConfig(&config.ReplicationMode, data) + updated, found, err := jsonutil.MergeJsonObject(&config.ReplicationMode, data) if err != nil { return err } @@ -229,12 +218,7 @@ func (h *confHandler) updateReplicationModeConfig(config *config.Config, key []s } func (h *confHandler) updatePDServerConfig(config *config.Config, key string, value interface{}) error { - data, err := json.Marshal(map[string]interface{}{key: value}) - if err != nil { - return err - } - - updated, found, err := mergeConfig(&config.PDServerCfg, data) + updated, found, err := jsonutil.AddKeyValue(&config.PDServerCfg, key, value) if err != nil { return err } @@ -286,23 +270,6 @@ func getConfigMap(cfg map[string]interface{}, key []string, value interface{}) m return cfg } -func mergeConfig(v interface{}, data []byte) (updated bool, found bool, err error) { - old, _ := json.Marshal(v) - if err := json.Unmarshal(data, v); err != nil { - return false, false, err - } - new, _ := json.Marshal(v) - if !bytes.Equal(old, new) { - return true, true, nil - } - m := make(map[string]interface{}) - if err := json.Unmarshal(data, &m); err != nil { - return false, false, err - } - found = reflectutil.FindSameFieldByJSON(v, m) - return false, found, nil -} - // @Tags config // @Summary Get schedule config. // @Produce json diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index c136f8fbf4e..968f5ae1b4d 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/pingcap/errors" + "github.com/tikv/pd/pkg/jsonutil" "github.com/tikv/pd/pkg/reflectutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/config" @@ -110,12 +111,7 @@ func (h *serviceMiddlewareHandler) updateServiceMiddlewareConfig(cfg *config.Ser } func (h *serviceMiddlewareHandler) updateAudit(config *config.ServiceMiddlewareConfig, key string, value interface{}) error { - data, err := json.Marshal(map[string]interface{}{key: value}) - if err != nil { - return err - } - - updated, found, err := mergeConfig(&config.AuditConfig, data) + updated, found, err := jsonutil.AddKeyValue(&config.AuditConfig, key, value) if err != nil { return err } From 392ce5a1effac4c5c86675bdbf01d52f825042cd Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 13 Jun 2022 15:55:52 +0800 Subject: [PATCH 39/56] add json merge util Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 30 ++---------------------------- server/server.go | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 2003ef23d80..949d98ca806 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -110,7 +110,7 @@ func (h *serviceMiddlewareHandler) updateServiceMiddlewareConfig(cfg *config.Ser case "audit": return h.updateAudit(cfg, kp[len(kp)-1], value) case "rate-limit": - return h.updateRateLimit(cfg, kp[len(kp)-1], value) + return h.svr.UpdateRateLimit(cfg, kp[len(kp)-1], value) } return errors.Errorf("config prefix %s not found", kp[0]) } @@ -131,32 +131,6 @@ func (h *serviceMiddlewareHandler) updateAudit(config *config.ServiceMiddlewareC return err } -func (h *serviceMiddlewareHandler) updateRateLimit(config *config.ServiceMiddlewareConfig, key string, value interface{}) error { - updated, found, err := jsonutil.AddKeyValue(&config.RateLimitConfig, key, value) - if err != nil { - return err - } - - if !found { - return errors.Errorf("config item %s not found", key) - } - - if updated { - err = h.svr.SetRateLimitConfig(config.RateLimitConfig) - } - return err -} - -func (h *serviceMiddlewareHandler) updateRateLimitConfig(key, label string, value ratelimit.DimensionConfig) error { - cfg := h.svr.GetServiceMiddlewareConfig() - rateLimitCfg := make(map[string]ratelimit.DimensionConfig) - for label, item := range cfg.LimiterConfig { - rateLimitCfg[label] = item - } - rateLimitCfg[label] = value - return h.updateRateLimit(cfg, key, &rateLimitCfg) -} - // @Tags service_middleware // @Summary update ratelimit config // @Param body body object string "json params" @@ -245,7 +219,7 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * case status&ratelimit.ConcurrencyDeleted != 0: concurrencyUpdatedFlag = "Concurrency limiter is deleted." } - err := h.updateRateLimitConfig("limiter-config", serviceLabel, cfg) + err := h.svr.UpdateRateLimitConfig("limiter-config", serviceLabel, cfg) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) } else { diff --git a/server/server.go b/server/server.go index c3a337ec4d3..725a96e407c 100644 --- a/server/server.go +++ b/server/server.go @@ -44,6 +44,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/etcdutil" "github.com/tikv/pd/pkg/grpcutil" + "github.com/tikv/pd/pkg/jsonutil" "github.com/tikv/pd/pkg/logutil" "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/syncutil" @@ -982,6 +983,32 @@ func (s *Server) SetAuditConfig(cfg config.AuditConfig) error { return nil } +func (s *Server) UpdateRateLimitConfig(key, label string, value ratelimit.DimensionConfig) error { + cfg := s.GetServiceMiddlewareConfig() + rateLimitCfg := make(map[string]ratelimit.DimensionConfig) + for label, item := range cfg.LimiterConfig { + rateLimitCfg[label] = item + } + rateLimitCfg[label] = value + return s.UpdateRateLimit(cfg, key, &rateLimitCfg) +} + +func (s *Server) UpdateRateLimit(config *config.ServiceMiddlewareConfig, key string, value interface{}) error { + updated, found, err := jsonutil.AddKeyValue(&config.RateLimitConfig, key, value) + if err != nil { + return err + } + + if !found { + return errors.Errorf("config item %s not found", key) + } + + if updated { + err = s.SetRateLimitConfig(config.RateLimitConfig) + } + return err +} + // GetRateLimitConfig gets the rate limit config information. func (s *Server) GetRateLimitConfig() *config.RateLimitConfig { return s.serviceMiddlewarePersistOptions.GetRateLimitConfig().Clone() From c332aae70f65341c9d44e1442373bcaf108bde8b Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 13 Jun 2022 16:09:39 +0800 Subject: [PATCH 40/56] add comment Signed-off-by: Cabinfever_B --- pkg/jsonutil/jsonutil.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/jsonutil/jsonutil.go b/pkg/jsonutil/jsonutil.go index d43643a73c2..f80e31906d2 100644 --- a/pkg/jsonutil/jsonutil.go +++ b/pkg/jsonutil/jsonutil.go @@ -21,6 +21,7 @@ import ( "github.com/tikv/pd/pkg/reflectutil" ) +// AddKeyValue is used to add a key value pair into `old` func AddKeyValue(old interface{}, key string, value interface{}) (updated bool, found bool, err error) { data, err := json.Marshal(map[string]interface{}{key: value}) if err != nil { @@ -29,6 +30,7 @@ func AddKeyValue(old interface{}, key string, value interface{}) (updated bool, return MergeJsonObject(old, data) } +// MergeJsonObject is used to merge a marshaled json object into v func MergeJsonObject(v interface{}, data []byte) (updated bool, found bool, err error) { old, _ := json.Marshal(v) if err := json.Unmarshal(data, v); err != nil { From 39c0868fec9f276fe552a404c52e6f6d3e97793b Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 13 Jun 2022 16:16:45 +0800 Subject: [PATCH 41/56] add comment Signed-off-by: Cabinfever_B --- pkg/jsonutil/jsonutil.go | 6 +++--- server/api/config.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/jsonutil/jsonutil.go b/pkg/jsonutil/jsonutil.go index f80e31906d2..c5ae2f378da 100644 --- a/pkg/jsonutil/jsonutil.go +++ b/pkg/jsonutil/jsonutil.go @@ -27,11 +27,11 @@ func AddKeyValue(old interface{}, key string, value interface{}) (updated bool, if err != nil { return false, false, err } - return MergeJsonObject(old, data) + return MergeJSONObject(old, data) } -// MergeJsonObject is used to merge a marshaled json object into v -func MergeJsonObject(v interface{}, data []byte) (updated bool, found bool, err error) { +// MergeJSONObject is used to merge a marshaled json object into v +func MergeJSONObject(v interface{}, data []byte) (updated bool, found bool, err error) { old, _ := json.Marshal(v) if err := json.Unmarshal(data, v); err != nil { return false, false, err diff --git a/server/api/config.go b/server/api/config.go index 724ff119198..e21a21a3c81 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -202,7 +202,7 @@ func (h *confHandler) updateReplicationModeConfig(config *config.Config, key []s if err != nil { return err } - updated, found, err := jsonutil.MergeJsonObject(&config.ReplicationMode, data) + updated, found, err := jsonutil.MergeJSONObject(&config.ReplicationMode, data) if err != nil { return err } From cc6bd2b1527c9b0922ab3d7dc390db5b28302aa0 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 13 Jun 2022 16:24:39 +0800 Subject: [PATCH 42/56] add comment Signed-off-by: Cabinfever_B --- server/server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/server.go b/server/server.go index 3fc6df233f0..f4130c88ad1 100644 --- a/server/server.go +++ b/server/server.go @@ -984,6 +984,7 @@ func (s *Server) SetAuditConfig(cfg config.AuditConfig) error { return nil } +// UpdateRateLimitConfig is used to update rate-limit config which will reserve old limiter-config func (s *Server) UpdateRateLimitConfig(key, label string, value ratelimit.DimensionConfig) error { cfg := s.GetServiceMiddlewareConfig() rateLimitCfg := make(map[string]ratelimit.DimensionConfig) @@ -994,6 +995,7 @@ func (s *Server) UpdateRateLimitConfig(key, label string, value ratelimit.Dimens return s.UpdateRateLimit(cfg, key, &rateLimitCfg) } +// UpdateRateLimit is used to update rate-limit config which will overwrite limiter-config func (s *Server) UpdateRateLimit(config *config.ServiceMiddlewareConfig, key string, value interface{}) error { updated, found, err := jsonutil.AddKeyValue(&config.RateLimitConfig, key, value) if err != nil { From 2c7c24ac85b74b15337fec46f252d000c350a4cc Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 13 Jun 2022 16:48:25 +0800 Subject: [PATCH 43/56] add test Signed-off-by: Cabinfever_B --- pkg/jsonutil/jsonutil_test.go | 66 +++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 pkg/jsonutil/jsonutil_test.go diff --git a/pkg/jsonutil/jsonutil_test.go b/pkg/jsonutil/jsonutil_test.go new file mode 100644 index 00000000000..bf66839ca11 --- /dev/null +++ b/pkg/jsonutil/jsonutil_test.go @@ -0,0 +1,66 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonutil + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +type testJSONStructLevel1 struct { + Name string `json:"name"` + Sub1 testJSONStructLevel2 `json:"sub1"` + Sub2 testJSONStructLevel2 `json:"sub2"` +} + +type testJSONStructLevel2 struct { + SubName string `json:"sub-name"` +} + +func TestJSONUtil(t *testing.T) { + t.Parallel() + re := require.New(t) + father := &testJSONStructLevel1{ + Name: "father", + } + son1 := &testJSONStructLevel2{ + SubName: "son1", + } + update, found, err := AddKeyValue(&father, "sub1", &son1) + re.NoError(err) + re.True(update) + re.True(found) + + son2 := &testJSONStructLevel2{ + SubName: "son2", + } + + update, found, err = AddKeyValue(father, "sub2", &son2) + re.NoError(err) + re.True(update) + re.True(found) + + update, found, err = AddKeyValue(father, "sub3", &son2) + re.NoError(err) + re.False(update) + re.False(found) + + update, found, err = AddKeyValue(father, "sub2", &son2) + re.NoError(err) + re.False(update) + re.True(found) + +} From c2f6f236063b96a27dc4345afdc6940cfeea1c5a Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 13 Jun 2022 16:54:41 +0800 Subject: [PATCH 44/56] add test Signed-off-by: Cabinfever_B --- pkg/jsonutil/jsonutil_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/jsonutil/jsonutil_test.go b/pkg/jsonutil/jsonutil_test.go index bf66839ca11..a046fbaf70a 100644 --- a/pkg/jsonutil/jsonutil_test.go +++ b/pkg/jsonutil/jsonutil_test.go @@ -62,5 +62,4 @@ func TestJSONUtil(t *testing.T) { re.NoError(err) re.False(update) re.True(found) - } From 79ada21cb48f53b668013b4218a6e63e843902ae Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 14 Jun 2022 11:04:22 +0800 Subject: [PATCH 45/56] fix Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 2 +- server/server.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 949d98ca806..8a604c273d0 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -110,7 +110,7 @@ func (h *serviceMiddlewareHandler) updateServiceMiddlewareConfig(cfg *config.Ser case "audit": return h.updateAudit(cfg, kp[len(kp)-1], value) case "rate-limit": - return h.svr.UpdateRateLimit(cfg, kp[len(kp)-1], value) + return h.svr.UpdateRateLimit(&cfg.RateLimitConfig, kp[len(kp)-1], value) } return errors.Errorf("config prefix %s not found", kp[0]) } diff --git a/server/server.go b/server/server.go index f4130c88ad1..81a72a5ff19 100644 --- a/server/server.go +++ b/server/server.go @@ -992,12 +992,12 @@ func (s *Server) UpdateRateLimitConfig(key, label string, value ratelimit.Dimens rateLimitCfg[label] = item } rateLimitCfg[label] = value - return s.UpdateRateLimit(cfg, key, &rateLimitCfg) + return s.UpdateRateLimit(&cfg.RateLimitConfig, key, &rateLimitCfg) } // UpdateRateLimit is used to update rate-limit config which will overwrite limiter-config -func (s *Server) UpdateRateLimit(config *config.ServiceMiddlewareConfig, key string, value interface{}) error { - updated, found, err := jsonutil.AddKeyValue(&config.RateLimitConfig, key, value) +func (s *Server) UpdateRateLimit(cfg *config.RateLimitConfig, key string, value interface{}) error { + updated, found, err := jsonutil.AddKeyValue(&cfg, key, value) if err != nil { return err } @@ -1007,7 +1007,7 @@ func (s *Server) UpdateRateLimit(config *config.ServiceMiddlewareConfig, key str } if updated { - err = s.SetRateLimitConfig(config.RateLimitConfig) + err = s.SetRateLimitConfig(*cfg) } return err } From 7161d5c0d0cab01647cd2932b81bec40efbf2d7b Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 14 Jun 2022 13:40:21 +0800 Subject: [PATCH 46/56] fix Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 8a604c273d0..133e1a060b1 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -137,6 +137,7 @@ func (h *serviceMiddlewareHandler) updateAudit(config *config.ServiceMiddlewareC // @Produce json // @Success 200 {string} string "" // @Failure 400 {string} string "" +// @Failure 500 {string} string "config item not found" // @Router /service-middleware/rate-limit/config [POST] func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r *http.Request) { var input map[string]interface{} @@ -193,12 +194,11 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * qps, okq := input["qps"].(float64) if okq { brust := 0 - if qps > 0 { - if int(qps) > 1 { - brust = int(qps) - } else { - brust = 1 - } + + if int(qps) > 1 { + brust = int(qps) + } else if qps > 0 { + brust = 1 } cfg.QPS = qps cfg.QPSBurst = brust From c74112686cdba49646aededb3f29b8ae9aafa58c Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 14 Jun 2022 13:41:13 +0800 Subject: [PATCH 47/56] fix Signed-off-by: Cabinfever_B --- server/api/service_middleware_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/service_middleware_test.go b/server/api/service_middleware_test.go index cde02788cc8..8c55a9c40a2 100644 --- a/server/api/service_middleware_test.go +++ b/server/api/service_middleware_test.go @@ -312,7 +312,7 @@ func (s *testRateLimitConfigSuite) TestConfigRateLimitSwitch(c *C) { } postData, err = json.Marshal(ms) c.Assert(err, IsNil) - c.Assert(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(c, http.StatusBadRequest), tu.StringEqual(c, "config item rate-limit not found")), IsNil) + c.Assert(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(c, http.StatusInternalServerError), tu.StringEqual(c, "config item rate-limit not found")), IsNil) c.Assert(failpoint.Enable("github.com/tikv/pd/server/config/persistServiceMiddlewareFail", "return(true)"), IsNil) ms = map[string]interface{}{ From 1fedfbc833afb1ad212df06dfa1fe1a8787c97e6 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 14 Jun 2022 14:04:25 +0800 Subject: [PATCH 48/56] fix Signed-off-by: Cabinfever_B --- server/api/service_middleware_test.go | 2 +- server/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/api/service_middleware_test.go b/server/api/service_middleware_test.go index 8c55a9c40a2..cde02788cc8 100644 --- a/server/api/service_middleware_test.go +++ b/server/api/service_middleware_test.go @@ -312,7 +312,7 @@ func (s *testRateLimitConfigSuite) TestConfigRateLimitSwitch(c *C) { } postData, err = json.Marshal(ms) c.Assert(err, IsNil) - c.Assert(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(c, http.StatusInternalServerError), tu.StringEqual(c, "config item rate-limit not found")), IsNil) + c.Assert(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(c, http.StatusBadRequest), tu.StringEqual(c, "config item rate-limit not found")), IsNil) c.Assert(failpoint.Enable("github.com/tikv/pd/server/config/persistServiceMiddlewareFail", "return(true)"), IsNil) ms = map[string]interface{}{ diff --git a/server/server.go b/server/server.go index 81a72a5ff19..0bd26d41a58 100644 --- a/server/server.go +++ b/server/server.go @@ -997,7 +997,7 @@ func (s *Server) UpdateRateLimitConfig(key, label string, value ratelimit.Dimens // UpdateRateLimit is used to update rate-limit config which will overwrite limiter-config func (s *Server) UpdateRateLimit(cfg *config.RateLimitConfig, key string, value interface{}) error { - updated, found, err := jsonutil.AddKeyValue(&cfg, key, value) + updated, found, err := jsonutil.AddKeyValue(cfg, key, value) if err != nil { return err } From e9ee0e235a13ac31cc01c1305292217bc388bec9 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 14 Jun 2022 16:11:44 +0800 Subject: [PATCH 49/56] fix Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 133e1a060b1..0bfae2be65d 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -223,7 +223,9 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) } else { - h.rd.JSON(w, http.StatusOK, fmt.Sprintf("%s %s", concurrencyUpdatedFlag, qpsRateUpdatedFlag)) + config, _ := json.Marshal(h.svr.GetServiceMiddlewareConfig().RateLimitConfig.LimiterConfig) + h.rd.JSON(w, http.StatusOK, fmt.Sprintf("%s\n%s\nCurrent Rate-limit Config:\n%s", + concurrencyUpdatedFlag, qpsRateUpdatedFlag, string(config))) } } } From dd70f46b9c0cfe1bf34b2e390b44206247ce4252 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 14 Jun 2022 16:41:19 +0800 Subject: [PATCH 50/56] fix Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 11 ++++++++--- server/api/service_middleware_test.go | 19 +++++++++++++------ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 0bfae2be65d..df8606e2c36 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -223,9 +223,14 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) } else { - config, _ := json.Marshal(h.svr.GetServiceMiddlewareConfig().RateLimitConfig.LimiterConfig) - h.rd.JSON(w, http.StatusOK, fmt.Sprintf("%s\n%s\nCurrent Rate-limit Config:\n%s", - concurrencyUpdatedFlag, qpsRateUpdatedFlag, string(config))) + result := rateLimitResult{concurrencyUpdatedFlag, qpsRateUpdatedFlag, h.svr.GetServiceMiddlewareConfig().RateLimitConfig.LimiterConfig} + h.rd.JSON(w, http.StatusOK, result) } } } + +type rateLimitResult struct { + ConcurrencyUpdatedFlag string `json:"Concurrency"` + QpsRateUpdatedFlag string `json:"qps"` + LimiterConfig map[string]ratelimit.DimensionConfig `json:"new-config"` +} diff --git a/server/api/service_middleware_test.go b/server/api/service_middleware_test.go index cde02788cc8..ff58df2992f 100644 --- a/server/api/service_middleware_test.go +++ b/server/api/service_middleware_test.go @@ -207,13 +207,13 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, - tu.StatusOK(c), tu.StringEqual(c, "Concurrency limiter is changed.")) + tu.StatusOK(c), tu.StringContain(c, "Concurrency limiter is changed.")) c.Assert(err, IsNil) input["concurrency"] = 0 jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, - tu.StatusOK(c), tu.StringEqual(c, "Concurrency limiter is deleted.")) + tu.StatusOK(c), tu.StringContain(c, "Concurrency limiter is deleted.")) c.Assert(err, IsNil) // change qps @@ -225,7 +225,7 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, - tu.StatusOK(c), tu.StringEqual(c, "QPS rate limiter is changed.")) + tu.StatusOK(c), tu.StringContain(c, "QPS rate limiter is changed.")) c.Assert(err, IsNil) input = make(map[string]interface{}) @@ -236,7 +236,7 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, - tu.StatusOK(c), tu.StringEqual(c, "QPS rate limiter is changed.")) + tu.StatusOK(c), tu.StringContain(c, "QPS rate limiter is changed.")) c.Assert(err, IsNil) c.Assert(s.svr.GetRateLimitConfig().LimiterConfig["GetHealthStatus"].QPSBurst, Equals, 1) @@ -244,7 +244,7 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, - tu.StatusOK(c), tu.StringEqual(c, "QPS rate limiter is deleted.")) + tu.StatusOK(c), tu.StringContain(c, "QPS rate limiter is deleted.")) c.Assert(err, IsNil) // change both @@ -255,8 +255,15 @@ func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { input["concurrency"] = 100 jsonBody, err = json.Marshal(input) c.Assert(err, IsNil) + result := rateLimitResult{} err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, - tu.StatusOK(c), tu.StringEqual(c, "\"Concurrency limiter is changed. QPS rate limiter is changed.\"\n")) + tu.StatusOK(c), tu.StringContain(c, "Concurrency limiter is changed."), + tu.StringContain(c, "QPS rate limiter is changed."), + tu.ExtractJSON(c, &result), + ) + c.Assert(result.LimiterConfig["Profile"].QPS, Equals, 100.) + c.Assert(result.LimiterConfig["Profile"].QPSBurst, Equals, 100) + c.Assert(result.LimiterConfig["Profile"].ConcurrencyLimit, Equals, uint64(100)) c.Assert(err, IsNil) limiter := s.svr.GetServiceRateLimiter() From b15ac5ab54f39e5cbe246757de847fd1f320d146 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 14 Jun 2022 16:48:16 +0800 Subject: [PATCH 51/56] fix type Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index df8606e2c36..6b4228013ad 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -231,6 +231,6 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * type rateLimitResult struct { ConcurrencyUpdatedFlag string `json:"Concurrency"` - QpsRateUpdatedFlag string `json:"qps"` + QPSRateUpdatedFlag string `json:"qps"` LimiterConfig map[string]ratelimit.DimensionConfig `json:"new-config"` } From 2d9f1246b8a797ce2e5968944b904558f276c08e Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 14 Jun 2022 16:49:54 +0800 Subject: [PATCH 52/56] fix typo Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 6b4228013ad..ae2ecbc81a5 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -231,6 +231,6 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * type rateLimitResult struct { ConcurrencyUpdatedFlag string `json:"Concurrency"` - QPSRateUpdatedFlag string `json:"qps"` + QPSRateUpdatedFlag string `json:"QPS"` LimiterConfig map[string]ratelimit.DimensionConfig `json:"new-config"` } From 97d1b9e16a20a6f5b4cb4da1f76012c89e9defac Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 14 Jun 2022 17:54:50 +0800 Subject: [PATCH 53/56] change url path Signed-off-by: Cabinfever_B --- server/api/router.go | 2 +- server/api/service_middleware.go | 6 +++--- server/api/service_middleware_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/api/router.go b/server/api/router.go index 61717d19ca0..3e8061fd74e 100644 --- a/server/api/router.go +++ b/server/api/router.go @@ -285,7 +285,7 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { serviceMiddlewareHandler := newServiceMiddlewareHandler(svr, rd) registerFunc(apiRouter, "/service-middleware/config", serviceMiddlewareHandler.GetServiceMiddlewareConfig, setMethods("GET")) registerFunc(apiRouter, "/service-middleware/config", serviceMiddlewareHandler.SetServiceMiddlewareConfig, setMethods("POST"), setAuditBackend(localLog)) - registerFunc(apiRouter, "/service-middleware/rate-limit/config", serviceMiddlewareHandler.SetRatelimitConfig, setMethods("POST"), setAuditBackend(localLog)) + registerFunc(apiRouter, "/service-middleware/config/rate-limit", serviceMiddlewareHandler.SetRatelimitConfig, setMethods("POST"), setAuditBackend(localLog)) logHandler := newLogHandler(svr, rd) registerFunc(apiRouter, "/admin/log", logHandler.SetLogLevel, setMethods("POST"), setAuditBackend(localLog)) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index ae2ecbc81a5..1f22a330621 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -138,7 +138,7 @@ func (h *serviceMiddlewareHandler) updateAudit(config *config.ServiceMiddlewareC // @Success 200 {string} string "" // @Failure 400 {string} string "" // @Failure 500 {string} string "config item not found" -// @Router /service-middleware/rate-limit/config [POST] +// @Router /service-middleware/config/rate-limit [POST] func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r *http.Request) { var input map[string]interface{} if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &input); err != nil { @@ -230,7 +230,7 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * } type rateLimitResult struct { - ConcurrencyUpdatedFlag string `json:"Concurrency"` - QPSRateUpdatedFlag string `json:"QPS"` + ConcurrencyUpdatedFlag string `json:"concurrency"` + QPSRateUpdatedFlag string `json:"qps"` LimiterConfig map[string]ratelimit.DimensionConfig `json:"new-config"` } diff --git a/server/api/service_middleware_test.go b/server/api/service_middleware_test.go index ff58df2992f..6ea0343f53b 100644 --- a/server/api/service_middleware_test.go +++ b/server/api/service_middleware_test.go @@ -129,7 +129,7 @@ func (s *testRateLimitConfigSuite) TearDownSuite(c *C) { } func (s *testRateLimitConfigSuite) TestUpdateRateLimitConfig(c *C) { - urlPrefix := fmt.Sprintf("%s%s/api/v1/service-middleware/rate-limit/config", s.svr.GetAddr(), apiPrefix) + urlPrefix := fmt.Sprintf("%s%s/api/v1/service-middleware/config/rate-limit", s.svr.GetAddr(), apiPrefix) // test empty type input := make(map[string]interface{}) From fa28dd49e0039434b6bc87ede2d7ffa97749ab0e Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 14 Jun 2022 23:26:42 +0800 Subject: [PATCH 54/56] fix Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 1f22a330621..72bbe1ec4e1 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -194,7 +194,6 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * qps, okq := input["qps"].(float64) if okq { brust := 0 - if int(qps) > 1 { brust = int(qps) } else if qps > 0 { From aca84d7e78b80942a0609adc49daa9939803d8ed Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 15 Jun 2022 11:56:39 +0800 Subject: [PATCH 55/56] fix Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index 72bbe1ec4e1..cb854b651d0 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -135,8 +135,8 @@ func (h *serviceMiddlewareHandler) updateAudit(config *config.ServiceMiddlewareC // @Summary update ratelimit config // @Param body body object string "json params" // @Produce json -// @Success 200 {string} string "" -// @Failure 400 {string} string "" +// @Success 200 {string} string +// @Failure 400 {string} string "The input is invalid." // @Failure 500 {string} string "config item not found" // @Router /service-middleware/config/rate-limit [POST] func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r *http.Request) { @@ -231,5 +231,5 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * type rateLimitResult struct { ConcurrencyUpdatedFlag string `json:"concurrency"` QPSRateUpdatedFlag string `json:"qps"` - LimiterConfig map[string]ratelimit.DimensionConfig `json:"new-config"` + LimiterConfig map[string]ratelimit.DimensionConfig `json:"limiter-config"` } From d5585759cbe8ffdc19fe91e9e9b1fdaa5c815749 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 15 Jun 2022 12:48:31 +0800 Subject: [PATCH 56/56] fix Signed-off-by: Cabinfever_B --- server/api/service_middleware.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/api/service_middleware.go b/server/api/service_middleware.go index cb854b651d0..426399a1d6e 100644 --- a/server/api/service_middleware.go +++ b/server/api/service_middleware.go @@ -186,8 +186,7 @@ func (h *serviceMiddlewareHandler) SetRatelimitConfig(w http.ResponseWriter, r * concurrencyUpdatedFlag := "Concurrency limiter is not changed." concurrencyFloat, okc := input["concurrency"].(float64) if okc { - concurrency := uint64(concurrencyFloat) - cfg.ConcurrencyLimit = concurrency + cfg.ConcurrencyLimit = uint64(concurrencyFloat) } // update qps rate limiter qpsRateUpdatedFlag := "QPS rate limiter is not changed."