Skip to content

Commit

Permalink
Merge pull request #13 from TencentBlueKing/develop
Browse files Browse the repository at this point in the history
- bufferd zap logger
- rate limit
  • Loading branch information
wklken committed Aug 4, 2021
2 parents c98c921 + a120f8d commit 3f4d8f4
Show file tree
Hide file tree
Showing 74 changed files with 1,556 additions and 400 deletions.
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.8.2
1.8.3
7 changes: 4 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ require (
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
github.com/alicebob/miniredis v2.5.0+incompatible
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/dlmiddlecote/sqlstats v1.0.2
github.com/elazarl/goproxy v0.0.0-20200710112657-153946a5f232 // indirect
github.com/fatih/structs v1.1.0
Expand All @@ -23,6 +22,7 @@ require (
github.com/go-redis/redis/v8 v8.10.0
github.com/go-sql-driver/mysql v1.6.0
github.com/gofrs/uuid v4.0.0+incompatible
github.com/golang-jwt/jwt/v4 v4.0.0
github.com/golang/mock v1.6.0
github.com/gomodule/redigo v1.8.2 // indirect
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
Expand Down Expand Up @@ -50,11 +50,12 @@ require (
github.com/ugorji/go v1.2.6 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.4
github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e // indirect
go.uber.org/atomic v1.8.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.7.0
go.uber.org/zap v1.17.0
go.uber.org/zap v1.18.1
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/time v0.0.0-20191024005414-555d28b269f0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
moul.io/http2curl v1.0.0
)
17 changes: 13 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down Expand Up @@ -112,7 +114,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
Expand Down Expand Up @@ -220,6 +221,8 @@ github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o=
github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
Expand Down Expand Up @@ -641,15 +644,18 @@ go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52l
go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.8.0 h1:CUhrE4N1rqSE6FM9ecihEjRkLQu8cDfgDyoOs83mEY4=
go.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4=
go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
Expand Down Expand Up @@ -690,6 +696,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
Expand Down Expand Up @@ -859,6 +866,7 @@ golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
Expand All @@ -882,6 +890,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
Expand Down
3 changes: 3 additions & 0 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ type Sentry struct {
// Quota ...
type Quota struct {
Model map[string]int

// NOTE: only used for rate limit middleware, will remove in the future
API map[string]int
}

// SystemQuota store the settings for specific system
Expand Down
8 changes: 7 additions & 1 deletion pkg/logging/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"fmt"
"strings"
"sync"
"time"

"github.com/sirupsen/logrus"
"go.uber.org/zap"
Expand Down Expand Up @@ -130,7 +131,12 @@ func newZapJSONLogger(cfg *config.LogConfig) *zap.Logger {
if err != nil {
panic(err)
}
w := zapcore.AddSync(writer)
//w := zapcore.AddSync(writer)
w := &zapcore.BufferedWriteSyncer{
WS: zapcore.AddSync(writer),
Size: 256 * 1024, // 256 kB
FlushInterval: 30 * time.Second,
}

// 设置日志级别
l, err := parseZapLogLevel(cfg.Level)
Expand Down
2 changes: 1 addition & 1 deletion pkg/middleware/client_jwt.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"errors"
"fmt"

"github.com/dgrijalva/jwt-go"
"github.com/golang-jwt/jwt/v4"

"iam/pkg/cache/impls"
)
Expand Down
2 changes: 1 addition & 1 deletion pkg/middleware/client_jwt_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
"testing"

"github.com/agiledragon/gomonkey"
"github.com/dgrijalva/jwt-go"
"github.com/golang-jwt/jwt/v4"
. "github.com/onsi/ginkgo"
"github.com/stretchr/testify/assert"

Expand Down
4 changes: 2 additions & 2 deletions pkg/middleware/logger.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func APILogger() gin.HandlerFunc {

return func(c *gin.Context) {
fields := logContextFields(c)
logger.With(fields...).Info("-")
logger.Info("-", fields...)
}
}

Expand All @@ -49,7 +49,7 @@ func WebLogger() gin.HandlerFunc {

return func(c *gin.Context) {
fields := logContextFields(c)
logger.With(fields...).Info("-")
logger.Info("-", fields...)
}
}

Expand Down
50 changes: 50 additions & 0 deletions pkg/middleware/ratelimit.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package middleware

import (
"sync"
"time"

"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
"golang.org/x/time/rate"

"iam/pkg/config"
"iam/pkg/util"
)

const (
rateLimitKey = "rate_limit"
// 2000 for per app_code per iam instance
defaultRateLimitPerSecondPerClient = 2000
)

var (
rateLimiters = sync.Map{}
)

// NOTE: this middleware used for api rate limit of calling directly
// all api will be maintained by APIGateway
// REMOVE THIS MIDDLEWARE WHEN WE NOT SUPPORT CALLING DIRECTLY

func NewRateLimitMiddleware(c *config.Config) gin.HandlerFunc {
limitCount, ok := c.Quota.API[rateLimitKey]
if !ok {
limitCount = defaultRateLimitPerSecondPerClient
}

return func(c *gin.Context) {
log.Debug("Middleware: RateLimit")

appCode := util.GetClientID(c)

value, _ := rateLimiters.LoadOrStore(appCode, rate.NewLimiter(rate.Every(1*time.Second), limitCount))
limiter := value.(*rate.Limiter)

if !limiter.Allow() {
util.TooManyRequestsJSONResponse(c, "hit the rate limit")
c.Abort()
return
}
c.Next()
}
}
3 changes: 3 additions & 0 deletions pkg/server/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,20 +57,23 @@ func NewRouter(cfg *config.Config) *gin.Engine {
policyRouter.Use(middleware.Metrics())
policyRouter.Use(middleware.APILogger())
policyRouter.Use(middleware.NewClientAuthMiddleware(cfg))
policyRouter.Use(middleware.NewRateLimitMiddleware(cfg))
policy.Register(policyRouter)

// restful apis for open api
openAPIRouter := router.Group("/api/v1/systems")
openAPIRouter.Use(middleware.Metrics())
openAPIRouter.Use(middleware.APILogger())
openAPIRouter.Use(middleware.NewClientAuthMiddleware(cfg))
policyRouter.Use(middleware.NewRateLimitMiddleware(cfg))
open.Register(openAPIRouter)

// perm-model for register
permModelRouter := router.Group("/api/v1/model")
permModelRouter.Use(middleware.Metrics())
permModelRouter.Use(middleware.Audit())
permModelRouter.Use(middleware.NewClientAuthMiddleware(cfg))
policyRouter.Use(middleware.NewRateLimitMiddleware(cfg))
model.Register(permModelRouter)

// debug api
Expand Down
5 changes: 5 additions & 0 deletions pkg/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
log "github.com/sirupsen/logrus"

"iam/pkg/config"
"iam/pkg/logging"
)

const (
Expand Down Expand Up @@ -116,6 +117,10 @@ func (s *Server) Stop() {
s.server.Close()
}

// flush logger
logging.GetAPILogger().Sync()
logging.GetWebLogger().Sync()

s.stopChan <- struct{}{}
}

Expand Down
1 change: 1 addition & 0 deletions pkg/util/error.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ const (
NotFoundError = 1901404
ConflictError = 1901409
SystemError = 1901500
TooManyRequests = 1901429
)

// ReportToSentry is a shortcut to build and send an event to sentry
Expand Down
1 change: 1 addition & 0 deletions pkg/util/response.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ var (
UnauthorizedJSONResponse = NewErrorJSONResponse(UnauthorizedError, "unauthorized")
NotFoundJSONResponse = NewErrorJSONResponse(NotFoundError, "not found")
ConflictJSONResponse = NewErrorJSONResponse(ConflictError, "conflict")
TooManyRequestsJSONResponse = NewErrorJSONResponse(TooManyRequests, "too many requests")
)

// SystemErrorJSONResponse ...
Expand Down
5 changes: 5 additions & 0 deletions release.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
# 1.8.3

- add: zap buffered logger
- add: rate limit for api

# 1.8.2

- bugfix: wrong config reference by web logger
Expand Down
13 changes: 0 additions & 13 deletions vendor/github.com/dgrijalva/jwt-go/.travis.yml

This file was deleted.

Loading

0 comments on commit 3f4d8f4

Please sign in to comment.