Merge pull request #1 from save95/develop

CHANGELOG.md

@@ -1,7 +1,39 @@
 # 变更日志
 
+## v1.2.0
+
+1. 增加 http 中间件：
+   - xss 过滤器： `middleware.XSSFilter()`
+   - cors 处理器：`middleware.CORS()`
+   - 有状态的 jwt 处理器：`middleware.JWTStatefulWith()`，`middleware.JWTStatefulWithout()`
+   - session 管理器：`middleware.SessionWithStore()` 方便分布式部署，指定 session 存储器
+   - http 缓存器：`middleware.HttpCache()`
+   - 角色控制器：`middleware.WithRole()`
+2. 增加 json logger 格式，为适应云日志采集(阿里云/腾讯云)
+3. 增加 db 内部特地错误检查工具：
+   - 唯一键冲突：`dberror.IsDuplicateEntry()`
+4. 增加 动态表格结构的 restful 响应：`restful.TableWithPagination()`
+5. 增加 [HTTPSQS](http://zyan.cc/httpsqs/) 客户端及相关队列：
+   - 客户端：`httpsqs.NewClient()`
+   - 消费者：`listener.NewHttpSQSConsumer()`
+     - 其他情形：
+     ```golang
+     err := listener.NewHttpSQSConsumer(handler).
+        WithContext(ctx).
+        WithLog(global.Log).
+        Consume()
+     ```
+   - 消费者包装器：`listener.HTTPSQS(handler)`
+6. 其他组件升级：
+   - DB 分页查询参数 `pager.Option{}` 支持 `Preload`
+
+⚠️注意：本次升级存在不向下兼容部分：
+   - http 中间件：
+     - `middleware.HttpLogger(restful.LogOption{})` => `middleware.HttpLogger(middleware.HttpLoggerOption{})`
+
+
 ## v1.1.0
 
-1. 升级 `gorm v2`，并增加 `dbutil.ConnectWith` 方法以适应非 mysql、sqlite 的数据库连接
+1. 升级 `gorm v2`，并增加 `dbutil.ConnectWith()` 方法以适应非 mysql、sqlite 的数据库连接
 
 ## v1.0.0

application/manager.go

@@ -19,7 +19,7 @@ type manager struct {
 // opts 支持设置日志、注册应用
 // 日志必须实现了 xlog.XLog 接口
 // 注册应用必须实现了 IApplication 接口
-func NewManager(opts ...interface{}) *manager {
+func NewManager(opts ...interface{}) IManager {
 	m := &manager{}
 
 	for _, opt := range opts {
@@ -44,8 +44,8 @@ func NewManager(opts ...interface{}) *manager {
 }
 
 // Register 注册应用
-func (m *manager) Register(application IApplication) {
-	m.apps = append(m.apps, application)
+func (m *manager) Register(app IApplication) {
+	m.apps = append(m.apps, app)
 }
 
 // Run 启动应用

application/application.go → application/types.go

@@ -5,3 +5,10 @@ type IApplication interface {
 	Start() error    // 启动
 	Shutdown() error // 关闭
 }
+
+type IManager interface {
+	// Register 注册应用
+	Register(app IApplication)
+	// Run 启动应用
+	Run()
+}

framework/dbcache/default_cache.go

@@ -12,12 +12,11 @@ import (
 	"github.com/eko/gocache/v2/store"
 	"github.com/go-redis/redis/v8"
 	"github.com/save95/go-pkg/model/pager"
-	"github.com/save95/go-utils/sliceutil"
 	"github.com/save95/xerror"
 	"golang.org/x/sync/singleflight"
 )
 
-var single singleflight.Group
+var single = singleflight.Group{}
 
 type dbCache struct {
 	cacheManager *cache.Cache
@@ -55,22 +54,35 @@ func (s *dbCache) Paginate(ctx context.Context, opt pager.Option,
 	kbs, _ := json.Marshal(opt)
 	k := strings.ToLower(fmt.Sprintf("%x", md5.Sum(kbs)))
 	key := fmt.Sprintf("%s:paginate:%s", s.name, k)
+	//if strings.Contains(s.name, "round") {
+	//	fmt.Printf("========== %s ============\n", key)
+	//}
 
 	jsonStr, err := s.getOrQuery(ctx, key, func() (interface{}, error) {
 		records, total, err := fun()
 		if nil != err {
 			return nil, err
 		}
 
-		slices, ok := sliceutil.ToAny(records)
-		if !ok {
-			slices = make([]interface{}, 0)
+		//slices, ok := sliceutil.ToAny(records)
+		//if !ok {
+		//	slices = make([]interface{}, 0)
+		//}
+		//
+		//return &PaginateResult{
+		//	Data:  slices,
+		//	Total: total,
+		//	Query: opt,
+		//}, nil
+
+		bs, err := json.Marshal(records)
+		if nil != err {
+			return nil, err
 		}
 
 		return &PaginateResult{
-			Data:  slices,
-			Total: total,
-			Query: opt,
+			DataBytes: bs,
+			Total:     total,
 		}, nil
 	})
 	if nil != err {
@@ -87,23 +99,23 @@ func (s *dbCache) Paginate(ctx context.Context, opt pager.Option,
 
 func (s *dbCache) First(ctx context.Context, id uint,
 	fun func() (interface{}, error),
-) (interface{}, error) {
+) (string, error) {
 	if id == 0 {
-		return nil, xerror.New("id error")
+		return "", xerror.New("id error")
 	}
 
 	key := fmt.Sprintf("%s:first:%d", s.name, id)
 	jsonStr, err := s.getOrQuery(ctx, key, fun)
 	if nil != err {
-		return nil, err
+		return "", err
 	}
 
-	var data interface{}
-	if err := json.Unmarshal([]byte(jsonStr), &data); nil != err {
-		return nil, xerror.Wrap(err, "data unmarshal failed")
-	}
+	//var data interface{}
+	//if err := json.Unmarshal([]byte(jsonStr), &data); nil != err {
+	//	return nil, xerror.Wrap(err, "data unmarshal failed")
+	//}
 
-	return data, nil
+	return jsonStr, nil
 }
 
 func (s *dbCache) Remember(ctx context.Context, key string,

framework/dbcache/types.go

@@ -20,7 +20,7 @@ type IDBCache interface {
 		ctx context.Context,
 		id uint,
 		fun func() (interface{}, error),
-	) (interface{}, error) //  按 id 查询数据
+	) (string, error) //  按 id 查询数据
 	Remember(
 		ctx context.Context,
 		key string,
@@ -34,7 +34,6 @@ type IDBCache interface {
 }
 
 type PaginateResult struct {
-	Query pager.Option
-	Data  []interface{}
-	Total uint
+	DataBytes []byte
+	Total     uint
 }

framework/dberror/error_code.go

@@ -0,0 +1,6 @@
+package dberror
+
+const (
+	errorCodeMySQLDuplicateEntry            = 1062
+	errorCodeMySQLDuplicateEntryWithKeyName = 1586
+)

framework/dberror/is.go

@@ -0,0 +1,18 @@
+package dberror
+
+import (
+	"github.com/go-sql-driver/mysql"
+)
+
+func IsDuplicateEntry(err error) bool {
+	//if sqliteErr, ok := err.(sqlite3.Error); ok {
+	//	return sqliteErr.ExtendedCode == sqlite3.ErrConstraintUnique ||
+	//		sqliteErr.ExtendedCode == sqlite3.ErrConstraintPrimaryKey
+	//} else
+	if mysqlErr, ok := err.(*mysql.MySQLError); ok {
+		return mysqlErr.Number == errorCodeMySQLDuplicateEntry ||
+			mysqlErr.Number == errorCodeMySQLDuplicateEntryWithKeyName
+	}
+
+	return false
+}

framework/logger/format_json.go

@@ -0,0 +1,25 @@
+package logger
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+)
+
+type formatJson struct {
+}
+
+func (lf *formatJson) Format(entry *logrus.Entry) ([]byte, error) {
+	logs := map[string]interface{}{
+		"time":    entry.Time.Format("2006-01-02 15:04:05.000"),
+		"level":   strings.ToUpper(entry.Level.String()),
+		"message": entry.Message,
+	}
+
+	if len(entry.Data) > 0 {
+		logs["data"] = entry.Data
+	}
+
+	return json.Marshal(logs)
+}

framework/logger/format.go → framework/logger/format_text.go

@@ -8,10 +8,10 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-type Format struct {
+type formatText struct {
 }
 
-func (lf *Format) Format(entry *logrus.Entry) ([]byte, error) {
+func (lf *formatText) Format(entry *logrus.Entry) ([]byte, error) {
 
 	out := &bytes.Buffer{}
 	if entry.Buffer != nil {

framework/logger/format_test.go → framework/logger/format_text_test.go

@@ -30,7 +30,7 @@ func TestFormat_Format(t *testing.T) {
 		Context: nil,
 	}
 
-	f := &Format{}
+	f := &formatText{}
 	b, e := f.Format(entry)
 	if nil != e {
 		t.Error(e)

framework/logger/format_type.go

@@ -0,0 +1,8 @@
+package logger
+
+type LogFormat int8
+
+const (
+	LogFormatText LogFormat = iota
+	LogFormatJson
+)

framework/logger/logger.go

@@ -13,11 +13,12 @@ import (
 )
 
 type logger struct {
-	path     string     // 日志存放路径
-	category string     // 日志分类
-	stack    xlog.Stack // 日志存储方式
-	level    xlog.Level // 日志等级
-	engine   xlog.XLog  // 日志引擎
+	path      string     // 日志存放路径
+	category  string     // 日志分类
+	stack     xlog.Stack // 日志存储方式
+	level     xlog.Level // 日志等级
+	engine    xlog.XLog  // 日志引擎
+	formatter logrus.Formatter
 
 	traceId string
 }
@@ -49,11 +50,11 @@ func NewDefaultTraceLogger(traceId string) xlog.XLogger {
 	return NewLoggerWithTraceId(traceId, defaultDir, defaultCategory, xlog.DailyStack)
 }
 
-func NewLogger(path, category string, stack xlog.Stack) xlog.XLogger {
-	return NewLoggerWithTraceId("", path, category, stack)
+func NewLogger(path, category string, stack xlog.Stack, opts ...func(logger2 *logger)) xlog.XLogger {
+	return NewLoggerWithTraceId("", path, category, stack, opts...)
 }
 
-func NewLoggerWithTraceId(traceId, path, category string, stack xlog.Stack) xlog.XLogger {
+func NewLoggerWithTraceId(traceId, path, category string, stack xlog.Stack, opts ...func(logger2 *logger)) xlog.XLogger {
 	logger := &logger{
 		category: defaultCategory,
 		traceId:  traceId,
@@ -68,6 +69,11 @@ func NewLoggerWithTraceId(traceId, path, category string, stack xlog.Stack) xlog
 	if err := logger.setStack(stack); nil != err {
 		fmt.Printf("logger setStack failed: %s\n", err.Error())
 	}
+
+	for _, opt := range opts {
+		opt(logger)
+	}
+
 	if err := logger.setEngine(); nil != err {
 		fmt.Printf("logger setEngine failed: %s\n", err.Error())
 	}
@@ -132,9 +138,13 @@ func (l *logger) setEngine() error {
 		return nil
 	}
 
+	if l.formatter == nil {
+		l.formatter = &formatText{}
+	}
+
 	// 初始化引擎
 	eg := logrus.New()
-	eg.SetFormatter(&Format{})
+	eg.SetFormatter(l.formatter)
 	eg.SetLevel(logrus.InfoLevel)
 	eg.SetOutput(os.Stdout)
 	l.engine = eg

framework/logger/option.go

@@ -0,0 +1,20 @@
+package logger
+
+import "github.com/sirupsen/logrus"
+
+func WithFormat(format LogFormat) func(*logger) {
+	return func(l *logger) {
+		switch format {
+		case LogFormatJson:
+			l.formatter = &formatJson{}
+		default:
+			l.formatter = &formatText{}
+		}
+	}
+}
+
+func WithFormatter(formatter logrus.Formatter) func(*logger) {
+	return func(l *logger) {
+		l.formatter = formatter
+	}
+}

go.mod

@@ -8,24 +8,31 @@ require (
 	github.com/eko/gocache/v2 v2.1.0
 	github.com/gin-contrib/cors v1.3.1
 	github.com/gin-contrib/sessions v0.0.4
-	github.com/gin-gonic/gin v1.7.4
+	github.com/gin-gonic/gin v1.7.7
 	github.com/go-redis/redis v6.15.9+incompatible
-	github.com/go-redis/redis/v8 v8.9.0
+	github.com/go-redis/redis/v8 v8.11.5
+	github.com/go-sql-driver/mysql v1.6.0
 	github.com/golang-jwt/jwt/v4 v4.0.0
 	github.com/hashicorp/go-version v1.2.1
 	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
 	github.com/lestrrat-go/strftime v1.0.4 // indirect
 	github.com/mattn/go-sqlite3 v1.14.8 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.22
 	github.com/pkg/errors v0.9.1
 	github.com/robfig/cron/v3 v3.0.1
-	github.com/save95/go-utils v1.0.3
+	github.com/save95/go-utils v1.0.4-0.20230628150415-96036c0da336
 	github.com/save95/xerror v1.1.0
 	github.com/save95/xlog v0.0.1
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.7.0
-	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
+	github.com/stretchr/testify v1.7.1
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	gorm.io/driver/mysql v1.1.1
 	gorm.io/driver/sqlite v1.1.4
 	gorm.io/gorm v1.21.12
 )
+
+//replace (
+//	github.com/save95/go-utils => /Users/royee/Develop/PoeticalSoft/save95/go-utils
+//)