Merge pull request #42 from Dot-Liu/main

新增nsqd及kafka之access日志插件

app/goku/worker.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"github.com/eolinker/eosc"
+	"github.com/eolinker/eosc/extends"
+	"github.com/eolinker/goku/drivers/auth/aksk"
+	"github.com/eolinker/goku/drivers/auth/apikey"
+	"github.com/eolinker/goku/drivers/auth/basic"
+	"github.com/eolinker/goku/drivers/auth/jwt"
+	"github.com/eolinker/goku/drivers/discovery/consul"
+	"github.com/eolinker/goku/drivers/discovery/eureka"
+	"github.com/eolinker/goku/drivers/discovery/nacos"
+	"github.com/eolinker/goku/drivers/discovery/static"
+	"github.com/eolinker/goku/drivers/output/fileoutput"
+	"github.com/eolinker/goku/drivers/output/httpoutput"
+	"github.com/eolinker/goku/drivers/output/kafka"
+	"github.com/eolinker/goku/drivers/output/nsq"
+	"github.com/eolinker/goku/drivers/output/syslog"
+	access_log "github.com/eolinker/goku/drivers/plugins/access-log"
+	"github.com/eolinker/goku/drivers/plugins/auth"
+	circuit_breaker "github.com/eolinker/goku/drivers/plugins/circuit-breaker"
+	"github.com/eolinker/goku/drivers/plugins/cors"
+	extra_params "github.com/eolinker/goku/drivers/plugins/extra-params"
+	"github.com/eolinker/goku/drivers/plugins/gzip"
+	ip_restriction "github.com/eolinker/goku/drivers/plugins/ip-restriction"
+	params_transformer "github.com/eolinker/goku/drivers/plugins/params-transformer"
+	proxy_rewrite "github.com/eolinker/goku/drivers/plugins/proxy-rewrite"
+	rate_limiting "github.com/eolinker/goku/drivers/plugins/rate-limiting"
+	response_rewrite "github.com/eolinker/goku/drivers/plugins/response-rewrite"
+	"github.com/eolinker/goku/drivers/plugins/rewrite"
+	http_router "github.com/eolinker/goku/drivers/router/http-router"
+	service_http "github.com/eolinker/goku/drivers/service/service-http"
+	upstream_http "github.com/eolinker/goku/drivers/upstream/upstream-http"
+	plugin_manager "github.com/eolinker/goku/plugin-manager"
+)
+
+func registerInnerExtenders() {
+	extends.AddInnerExtendProject("eolinker.com", "goku", Register)
+}
+func Register(extenderRegister eosc.IExtenderDriverRegister) {
+	// router
+	http_router.Register(extenderRegister)
+
+	// service
+	service_http.Register(extenderRegister)
+
+	// upstream
+	upstream_http.Register(extenderRegister)
+
+	// discovery
+	static.Register(extenderRegister)
+	nacos.Register(extenderRegister)
+	consul.Register(extenderRegister)
+	eureka.Register(extenderRegister)
+	//output
+	fileoutput.Register(extenderRegister)
+	nsq.Register(extenderRegister)
+	httpoutput.Register(extenderRegister)
+	kafka.Register(extenderRegister)
+	syslog.Register(extenderRegister)
+	// auth
+	basic.Register(extenderRegister)
+	apikey.Register(extenderRegister)
+	aksk.Register(extenderRegister)
+	jwt.Register(extenderRegister)
+
+	//plugin
+	plugin_manager.Register(extenderRegister)
+	auth.Register(extenderRegister)
+	rewrite.Register(extenderRegister)
+
+	extra_params.Register(extenderRegister)
+	params_transformer.Register(extenderRegister)
+	proxy_rewrite.Register(extenderRegister)
+	ip_restriction.Register(extenderRegister)
+	rate_limiting.Register(extenderRegister)
+	cors.Register(extenderRegister)
+	gzip.Register(extenderRegister)
+	response_rewrite.Register(extenderRegister)
+	circuit_breaker.Register(extenderRegister)
+
+	access_log.Register(extenderRegister)
+}

build/resources/config.yml.tmp

@@ -1,8 +1,4 @@
 listen:   # node listen port
-<<<<<<< HEAD:build/resources/config.yml
-=======
-# - 80
->>>>>>> origin/feature/rename:build/resources/config.yml.tmp
   - 8080
 
 admin:    # openAPI request info

drivers/output/fileoutput/driver.go

@@ -14,42 +14,44 @@ func (d *Driver) ConfigType() reflect.Type {
 	return d.configType
 }
 
-
 func (d *Driver) Check(v interface{}) (*Config, error) {
 	conf, ok := v.(*Config)
 	if !ok {
 		return nil, errorConfigType
 	}
 
 	fileConf := conf.Config
-	if fileConf == nil{
+	if fileConf == nil {
 		return nil, errorNilConfig
 	}
 
 	if fileConf.Dir == "" {
-		return nil,errorConfDir
+		return nil, errorConfDir
 	}
-	if fileConf.File == ""{
-		return nil,errorConfFile
+	if fileConf.File == "" {
+		return nil, errorConfFile
 	}
-	if fileConf.Period != "day" && fileConf.Period != "hour"{
-		return nil,errorConfPeriod
+	if fileConf.Period != "day" && fileConf.Period != "hour" {
+		return nil, errorConfPeriod
 	}
 
 	if fileConf.Expire == 0 {
 		fileConf.Expire = 3
 	}
-	if fileConf.Type == ""{
+	if fileConf.Type == "" {
 		fileConf.Type = "line"
 	}
 
+	if len(fileConf.Formatter) == 0 {
+		return nil, errFormatterConf
+	}
 	return conf, nil
 }
 
 func (d *Driver) Create(id, name string, v interface{}, workers map[eosc.RequireId]interface{}) (eosc.IWorker, error) {
 	worker := &FileOutput{
-		Driver:d,
-		id: id,
+		Driver: d,
+		id:     id,
 	}
 	err := worker.Reset(v, workers)
 	return worker, err

drivers/output/fileoutput/error.go

@@ -5,8 +5,9 @@ import "errors"
 var (
 	errorConfigType    = errors.New("error config type")
 	errorFormatterType = errors.New("error formatter type")
-	errorNilConfig = errors.New("error nil config")
-	errorConfDir = errors.New("error dir is illegal")
-	errorConfFile = errors.New("error file is illegal")
-	errorConfPeriod = errors.New("error period is illegal")
+	errorNilConfig     = errors.New("error nil config")
+	errorConfDir       = errors.New("error dir is illegal")
+	errorConfFile      = errors.New("error file is illegal")
+	errorConfPeriod    = errors.New("error period is illegal")
+	errFormatterConf   = errors.New("formatter config can not be null. ")
 )

drivers/output/httpoutput/config.go

@@ -0,0 +1,61 @@
+package httpoutput
+
+import (
+	"encoding/json"
+	"github.com/eolinker/eosc"
+	"net/http"
+)
+
+type Config struct {
+	Config *HttpConf `json:"config" yaml:"config"`
+}
+
+type HttpConf struct {
+	Method    string               `json:"method" yaml:"method"`
+	Url       string               `json:"url" yaml:"url"`
+	Headers   map[string]string    `json:"headers" yaml:"headers"`
+	Type      string               `json:"type" yaml:"type"`
+	Formatter eosc.FormatterConfig `json:"formatter" yaml:"formatter"`
+}
+
+func (h *HttpConf) isConfUpdate(conf *HttpConf) bool {
+	if h.Method != conf.Method || h.Url != conf.Url || !compareTwoMapStringEqual(h.Headers, conf.Headers) {
+		return true
+	}
+	return false
+}
+
+func compareTwoMapStringEqual(mapA, mapB map[string]string) bool {
+	if len(mapA) != len(mapB) {
+		return false
+	}
+	length := len(mapA)
+	keySlice := make([]string, 0, length)
+	dataValueA := make([]string, 0, length)
+	dataValueB := make([]string, 0, length)
+	for k, v := range mapA {
+		keySlice = append(keySlice, k)
+		dataValueA = append(dataValueA, v)
+	}
+
+	for _, key := range keySlice {
+		if vb, has := mapB[key]; has {
+			dataValueB = append(dataValueB, vb)
+		} else {
+			return false
+		}
+	}
+
+	strValueA, _ := json.Marshal(dataValueA)
+	strValueB, _ := json.Marshal(dataValueB)
+
+	return string(strValueA) == string(strValueB)
+}
+
+func toHeader(items map[string]string) http.Header {
+	header := make(http.Header)
+	for k, v := range items {
+		header.Set(k, v)
+	}
+	return header
+}

drivers/output/httpoutput/driver.go

@@ -0,0 +1,89 @@
+package httpoutput
+
+import (
+	"github.com/eolinker/eosc/formatter"
+	http_transport "github.com/eolinker/goku/output/http-transport"
+	"reflect"
+
+	"github.com/eolinker/eosc"
+)
+
+type Driver struct {
+	configType reflect.Type
+}
+
+func (d *Driver) ConfigType() reflect.Type {
+	return d.configType
+}
+
+func (d *Driver) Check(v interface{}) (*HttpConf, error) {
+	conf, ok := v.(*Config)
+	if !ok {
+		return nil, errConfigType
+	}
+
+	httpConf := conf.Config
+	if httpConf.Method == "" {
+		return nil, errMethod
+	}
+	switch httpConf.Method {
+	case "GET", "POST", "HEAD", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE":
+	default:
+		return nil, errMethod
+	}
+
+	if httpConf.Url == "" {
+		return nil, errUrlNull
+	}
+
+	if httpConf.Type == "" {
+		httpConf.Type = "line"
+	}
+
+	switch httpConf.Type {
+	case "line", "json":
+	default:
+		return nil, errFormatterType
+	}
+
+	if len(httpConf.Formatter) == 0 {
+		return nil, errFormatterConf
+	}
+
+	return httpConf, nil
+}
+
+func (d *Driver) Create(id, name string, v interface{}, workers map[eosc.RequireId]interface{}) (eosc.IWorker, error) {
+	worker := &HttpOutput{
+		Driver: d,
+		id:     id,
+	}
+
+	conf, err := d.Check(v)
+	if err != nil {
+		return nil, err
+	}
+
+	worker.config = conf
+
+	cfg := &http_transport.Config{
+		Method:       conf.Method,
+		Url:          conf.Url,
+		Headers:      toHeader(conf.Headers),
+		HandlerCount: 5, // 默认值， 以后可能会改成配置
+	}
+
+	worker.transport, err = http_transport.CreateTransporter(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	//创建formatter
+	factory, has := formatter.GetFormatterFactory(conf.Type)
+	if !has {
+		return nil, errFormatterType
+	}
+	worker.formatter, err = factory.Create(conf.Formatter)
+
+	return worker, err
+}

drivers/output/httpoutput/error.go

@@ -0,0 +1,11 @@
+package httpoutput
+
+import "errors"
+
+var (
+	errConfigType    = errors.New("config type does not match. ")
+	errMethod        = errors.New("method is illegal. ")
+	errUrlNull       = errors.New("url can not be null. ")
+	errFormatterType = errors.New("type is illegal. ")
+	errFormatterConf = errors.New("formatter config can not be null. ")
+)

drivers/output/httpoutput/factory.go

@@ -0,0 +1,26 @@
+package httpoutput
+
+import (
+	"github.com/eolinker/eosc"
+	"reflect"
+)
+
+const name = "http_output"
+
+//Register 注册file_output驱动工厂
+func Register(register eosc.IExtenderDriverRegister) {
+	register.RegisterExtenderDriver(name, NewFactory())
+}
+
+type Factory struct {
+}
+
+func NewFactory() *Factory {
+	return &Factory{}
+}
+
+func (f *Factory) Create(profession string, name string, label string, desc string, params map[string]interface{}) (eosc.IExtenderDriver, error) {
+	return &Driver{
+		configType: reflect.TypeOf((*Config)(nil)),
+	}, nil
+}