diff --git a/docs/en/deployment/configuration/manager.yaml b/docs/en/deployment/configuration/manager.yaml index 7f316ec00ec..d1b61d0b41d 100644 --- a/docs/en/deployment/configuration/manager.yaml +++ b/docs/en/deployment/configuration/manager.yaml @@ -25,6 +25,16 @@ database: host: dragonfly port: 3306 dbname: manager + migrate: true + # tls: + # # client certificate file path + # cert: /etc/ssl/certs/cert.pem + # # client key file path + # key: /etc/ssl/private/key.pem + # # ca file path + # ca: /etc/ssl/certs/ca.pem + # # whether a client verifies the server's certificate chain and host name. + # insecureSkipVerify: true # redis configure redis: password: dragonfly diff --git a/docs/zh-CN/deployment/configuration/manager.yaml b/docs/zh-CN/deployment/configuration/manager.yaml index b4c87cbca69..dbefdffa1a1 100644 --- a/docs/zh-CN/deployment/configuration/manager.yaml +++ b/docs/zh-CN/deployment/configuration/manager.yaml @@ -24,6 +24,16 @@ database: host: dragonfly port: 3306 dbname: manager + migrate: true + # tls: + # # 客户端证书文件路径 + # cert: /etc/ssl/certs/cert.pem + # # 客户端私钥文件路径 + # key: /etc/ssl/private/key.pem + # # CA 证书文件路径 + # ca: /etc/ssl/certs/ca.pem + # # 客户端是否验证服务端的证书链和 hostname + # insecureSkipVerify: true # redis 配置 redis: password: dragonfly diff --git a/go.mod b/go.mod index 75529c04305..24ef1fec211 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/casbin/gorm-adapter/v3 v3.3.2 github.com/colinmarc/hdfs/v2 v2.2.0 github.com/distribution/distribution/v3 v3.0.0-20210804104954-38ab4c606ee3 + github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 github.com/emirpasic/gods v1.12.0 github.com/envoyproxy/protoc-gen-validate v0.6.1 @@ -42,6 +43,7 @@ require ( github.com/mcuadros/go-gin-prometheus v0.1.0 github.com/mitchellh/mapstructure v1.4.1 github.com/montanaflynn/stats v0.6.6 + github.com/onsi/ginkgo/v2 v2.1.0 github.com/onsi/gomega v1.18.0 github.com/opencontainers/go-digest v1.0.0 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 @@ -153,8 +155,6 @@ require ( github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/nxadm/tail v1.4.8 // indirect - github.com/onsi/ginkgo/v2 v2.1.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pelletier/go-toml v1.8.1 // indirect @@ -198,7 +198,6 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea // indirect gopkg.in/ini.v1 v1.51.0 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gorm.io/driver/postgres v1.0.8 // indirect gorm.io/driver/sqlserver v1.0.4 // indirect diff --git a/go.sum b/go.sum index 82dcc0cdd3b..deaf9e25be6 100644 --- a/go.sum +++ b/go.sum @@ -152,6 +152,8 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/distribution/distribution/v3 v3.0.0-20210804104954-38ab4c606ee3 h1:rEK0juuU5idazw//KzUcL3yYwUU3DIe2OnfJwjDBqno= github.com/distribution/distribution/v3 v3.0.0-20210804104954-38ab4c606ee3/go.mod h1:gt38b7cvVKazi5XkHvINNytZXgTEntyhtyM3HQz46Nk= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= @@ -647,9 +649,8 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.0 h1:Rj+Was8Gt/ICSZY/CihVKK2cGy3rDqBYKSxVnmOXZzI= github.com/onsi/ginkgo/v2 v2.1.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= @@ -660,9 +661,6 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= -github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI= -github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.0 h1:ngbYoRctxjl8SiF7XgP0NxBFbfHcg3wfHMMaFHWwMTM= github.com/onsi/gomega v1.18.0/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= @@ -1125,8 +1123,6 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/manager/config/config.go b/manager/config/config.go index b53faaa95de..664595b8230 100644 --- a/manager/config/config.go +++ b/manager/config/config.go @@ -17,66 +17,148 @@ package config import ( + "crypto/tls" "errors" "time" + "github.com/docker/go-connections/tlsconfig" + "d7y.io/dragonfly/v2/cmd/dependency/base" ) type Config struct { + // Base options base.Options `yaml:",inline" mapstructure:",squash"` - Server *ServerConfig `yaml:"server" mapstructure:"server"` - Database *DatabaseConfig `yaml:"database" mapstructure:"database"` - Cache *CacheConfig `yaml:"cache" mapstructure:"cache"` - Metrics *RestConfig `yaml:"metrics" mapstructure:"metrics"` + + // Server configuration + Server *ServerConfig `yaml:"server" mapstructure:"server"` + + // Database configuration + Database *DatabaseConfig `yaml:"database" mapstructure:"database"` + + // Cache configuration + Cache *CacheConfig `yaml:"cache" mapstructure:"cache"` + + // Metrics configuration + Metrics *RestConfig `yaml:"metrics" mapstructure:"metrics"` } type ServerConfig struct { - Name string `yaml:"name" mapstructure:"name"` - LogDir string `yaml:"logDir" mapstructure:"logDir"` - PublicPath string `yaml:"publicPath" mapstructure:"publicPath"` - GRPC *TCPListenConfig `yaml:"grpc" mapstructure:"grpc"` - REST *RestConfig `yaml:"rest" mapstructure:"rest"` + // Server name + Name string `yaml:"name" mapstructure:"name"` + + // Server log directory + LogDir string `yaml:"logDir" mapstructure:"logDir"` + + // Console resource path + PublicPath string `yaml:"publicPath" mapstructure:"publicPath"` + + // GRPC server configuration + GRPC *TCPListenConfig `yaml:"grpc" mapstructure:"grpc"` + + // REST server configuration + REST *RestConfig `yaml:"rest" mapstructure:"rest"` } type DatabaseConfig struct { + // Mysql configuration Mysql *MysqlConfig `yaml:"mysql" mapstructure:"mysql"` + + // Redis configuration Redis *RedisConfig `yaml:"redis" mapstructure:"redis"` } type MysqlConfig struct { - User string `yaml:"user" mapstructure:"user"` + // Server username + User string `yaml:"user" mapstructure:"user"` + + // Server password Password string `yaml:"password" mapstructure:"password"` - Host string `yaml:"host" mapstructure:"host"` - Port int `yaml:"port" mapstructure:"port"` - DBName string `yaml:"dbname" mapstructure:"dbname"` - Migrate bool `yaml:"migrate" mapstructure:"migrate"` + + // Server host + Host string `yaml:"host" mapstructure:"host"` + + // Server port + Port int `yaml:"port" mapstructure:"port"` + + // Server DB name + DBName string `yaml:"dbname" mapstructure:"dbname"` + + // Enable migration + Migrate bool `yaml:"migrate" mapstructure:"migrate"` + + // TLS configuration + TLS *TLSConfig `yaml:"tls" mapstructure:"tls"` +} + +type TLSConfig struct { + // Client certificate file path + Cert string `yaml:"cert" mapstructure:"cert"` + + // Client key file path + Key string `yaml:"key" mapstructure:"key"` + + // CA file path + CA string `yaml:"ca" mapstructure:"ca"` + + // InsecureSkipVerify controls whether a client verifies the + // server's certificate chain and host name. + InsecureSkipVerify bool `yaml:"insecureSkipVerify" mapstructure:"insecureSkipVerify"` +} + +// Generate client tls config +func (t *TLSConfig) Client() (*tls.Config, error) { + return tlsconfig.Client(tlsconfig.Options{ + CAFile: t.CA, + CertFile: t.Cert, + KeyFile: t.Key, + InsecureSkipVerify: t.InsecureSkipVerify, + }) } type RedisConfig struct { - Host string `yaml:"host" mapstructure:"host"` - Port int `yaml:"port" mapstructure:"port"` - Password string `yaml:"password" mapstructure:"password"` - CacheDB int `yaml:"cacheDB" mapstructure:"cacheDB"` - BrokerDB int `yaml:"brokerDB" mapstructure:"brokerDB"` - BackendDB int `yaml:"backendDB" mapstructure:"backendDB"` + // Server host + Host string `yaml:"host" mapstructure:"host"` + + // Server port + Port int `yaml:"port" mapstructure:"port"` + + // Server password + Password string `yaml:"password" mapstructure:"password"` + + // Server cache DB name + CacheDB int `yaml:"cacheDB" mapstructure:"cacheDB"` + + // Server broker DB name + BrokerDB int `yaml:"brokerDB" mapstructure:"brokerDB"` + + // Server backend DB name + BackendDB int `yaml:"backendDB" mapstructure:"backendDB"` } type CacheConfig struct { + // Redis cache configuration Redis *RedisCacheConfig `yaml:"redis" mapstructure:"redis"` + + // Local cache configuration Local *LocalCacheConfig `yaml:"local" mapstructure:"local"` } type RedisCacheConfig struct { + // Cache TTL TTL time.Duration `yaml:"ttl" mapstructure:"ttl"` } type LocalCacheConfig struct { - Size int `yaml:"size" mapstructure:"size"` - TTL time.Duration `yaml:"ttl" mapstructure:"ttl"` + // Size of LFU cache + Size int `yaml:"size" mapstructure:"size"` + + // Cache TTL + TTL time.Duration `yaml:"ttl" mapstructure:"ttl"` } type RestConfig struct { + // REST server address Addr string `yaml:"addr" mapstructure:"addr"` } @@ -93,6 +175,7 @@ type TCPListenPortRange struct { End int } +// New config instance func New() *Config { return &Config{ Server: &ServerConfig{ @@ -130,58 +213,116 @@ func New() *Config { } } +// Validate config values func (cfg *Config) Validate() error { + if cfg.Server == nil { + return errors.New("empty server config is not specified") + } + if cfg.Server.Name == "" { return errors.New("empty server name config is not specified") } - if cfg.Cache == nil { - return errors.New("empty cache config is not specified") + if cfg.Server.GRPC == nil { + return errors.New("empty grpc server config is not specified") + } + + if cfg.Server.REST == nil { + return errors.New("empty rest server config is not specified") } - if cfg.Cache != nil { - if cfg.Cache.Redis.TTL == 0 { - return errors.New("empty redis cache TTL is not specified") + if cfg.Database == nil { + return errors.New("empty database config is not specified") + } + + if cfg.Database.Redis == nil { + return errors.New("empty database redis config is not specified") + } + + if cfg.Database.Redis.Host == "" { + return errors.New("empty database redis host is not specified") + } + + if cfg.Database.Redis.Port <= 0 { + return errors.New("empty database redis port is not specified") + } + + if cfg.Database.Redis.CacheDB < 0 { + return errors.New("empty database redis cacheDB is not specified") + } + + if cfg.Database.Redis.BrokerDB < 0 { + return errors.New("empty database redis brokerDB is not specified") + } + + if cfg.Database.Redis.BackendDB < 0 { + return errors.New("empty database redis backendDB is not specified") + } + + if cfg.Database.Mysql == nil { + return errors.New("empty database mysql config is not specified") + } + + if cfg.Database.Mysql.User == "" { + return errors.New("empty database mysql user is not specified") + } + + if cfg.Database.Mysql.Password == "" { + return errors.New("empty database mysql password is not specified") + } + + if cfg.Database.Mysql.Host == "" { + return errors.New("empty database mysql host is not specified") + } + + if cfg.Database.Mysql.Port <= 0 { + return errors.New("empty database mysql port is not specified") + } + + if cfg.Database.Mysql.DBName == "" { + return errors.New("empty database mysql dbName is not specified") + } + + if cfg.Database.Mysql.TLS != nil { + if cfg.Database.Mysql.TLS.Cert == "" { + return errors.New("empty database mysql tls cert is not specified") } - if cfg.Cache.Local.Size == 0 { - return errors.New("empty local cache size is not specified") + if cfg.Database.Mysql.TLS.Key == "" { + return errors.New("empty database mysql tls key is not specified") } - if cfg.Cache.Local.TTL == 0 { - return errors.New("empty local cache TTL is not specified") + if cfg.Database.Mysql.TLS.CA == "" { + return errors.New("empty database mysql tls ca is not specified") } } - if cfg.Database == nil { - return errors.New("empty mysql config is not specified") + if cfg.Cache == nil { + return errors.New("empty cache config is not specified") } - if cfg.Database != nil { - if cfg.Database.Redis.Host == "" { - return errors.New("empty cache redis config is not specified") - } + if cfg.Cache.Redis == nil { + return errors.New("empty cache redis config is not specified") + } - if cfg.Database.Mysql == nil { - if cfg.Database.Mysql.Host == "" { - return errors.New("empty cache mysql host is not specified") - } - return errors.New("empty cache mysql config is not specified") - } + if cfg.Cache.Redis.TTL == 0 { + return errors.New("empty cache redis TTL is not specified") } - if cfg.Server == nil { - return errors.New("empty server config is not specified") + if cfg.Cache.Local == nil { + return errors.New("empty cache local config is not specified") } - if cfg.Server != nil { - if cfg.Server.GRPC == nil { - return errors.New("empty grpc server config is not specified") - } + if cfg.Cache.Local.Size == 0 { + return errors.New("empty cache local size is not specified") + } - if cfg.Server.REST == nil { - return errors.New("empty rest server config is not specified") - } + if cfg.Cache.Local.TTL == 0 { + return errors.New("empty cache local TTL is not specified") + } + + if cfg.Metrics != nil && cfg.Metrics.Addr == "" { + return errors.New("empty metrics addr is not specified") } return nil diff --git a/manager/config/config_test.go b/manager/config/config_test.go index a5d57392fe5..7e1a7362546 100644 --- a/manager/config/config_test.go +++ b/manager/config/config_test.go @@ -51,6 +51,12 @@ func TestManagerConfig_Load(t *testing.T) { Host: "foo", Port: 3306, DBName: "foo", + TLS: &TLSConfig{ + Cert: "foo", + Key: "foo", + CA: "foo", + InsecureSkipVerify: true, + }, }, Redis: &RedisConfig{ Host: "bar", diff --git a/manager/config/testdata/manager.yaml b/manager/config/testdata/manager.yaml index 8bf968a01b6..c78e158a80e 100644 --- a/manager/config/testdata/manager.yaml +++ b/manager/config/testdata/manager.yaml @@ -17,6 +17,11 @@ database: host: foo port: 3306 dbname: foo + tls: + cert: foo + key: foo + ca: foo + insecureSkipVerify: true redis: password: bar host: bar diff --git a/manager/database/database.go b/manager/database/database.go index 698e3de4dc8..5450944d4b7 100644 --- a/manager/database/database.go +++ b/manager/database/database.go @@ -17,23 +17,10 @@ package database import ( - "context" - "fmt" - "github.com/go-redis/redis/v8" - "gorm.io/driver/mysql" "gorm.io/gorm" - "gorm.io/gorm/schema" - "moul.io/zapgorm2" - logger "d7y.io/dragonfly/v2/internal/dflog" "d7y.io/dragonfly/v2/manager/config" - "d7y.io/dragonfly/v2/manager/model" -) - -const ( - defaultCDNLoadLimit = 300 - defaultClientLoadLimit = 100 ) type Database struct { @@ -57,126 +44,3 @@ func New(cfg *config.Config) (*Database, error) { RDB: rdb, }, nil } - -func NewRedis(cfg *config.RedisConfig) (*redis.Client, error) { - redis.SetLogger(&redisLogger{}) - - client := redis.NewClient(&redis.Options{ - Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), - Password: cfg.Password, - DB: cfg.CacheDB, - }) - - if err := client.Ping(context.Background()).Err(); err != nil { - return nil, err - } - - return client, nil -} - -func newMyqsl(cfg *config.MysqlConfig) (*gorm.DB, error) { - logger := zapgorm2.New(logger.CoreLogger.Desugar()) - - dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local&interpolateParams=true", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.DBName) - dialector := mysql.Open(dsn) - db, err := gorm.Open(dialector, &gorm.Config{ - NamingStrategy: schema.NamingStrategy{ - SingularTable: true, - }, - DisableForeignKeyConstraintWhenMigrating: true, - Logger: logger, - }) - if err != nil { - return nil, err - } - - // Run migration - if cfg.Migrate { - if err := migrate(db); err != nil { - return nil, err - } - } - - // Run seed - if err := seed(db); err != nil { - return nil, err - } - - return db, nil -} - -func migrate(db *gorm.DB) error { - return db.Set("gorm:table_options", "DEFAULT CHARSET=utf8mb4 ROW_FORMAT=Dynamic").AutoMigrate( - &model.Job{}, - &model.CDNCluster{}, - &model.CDN{}, - &model.SchedulerCluster{}, - &model.Scheduler{}, - &model.SecurityRule{}, - &model.SecurityGroup{}, - &model.User{}, - &model.Oauth{}, - &model.Config{}, - &model.Application{}, - ) -} - -func seed(db *gorm.DB) error { - var cdnClusterCount int64 - if err := db.Model(model.CDNCluster{}).Count(&cdnClusterCount).Error; err != nil { - return err - } - if cdnClusterCount <= 0 { - if err := db.Create(&model.CDNCluster{ - Model: model.Model{ - ID: uint(1), - }, - Name: "cdn-cluster-1", - Config: map[string]interface{}{ - "load_limit": defaultCDNLoadLimit, - }, - IsDefault: true, - }).Error; err != nil { - return err - } - } - - var schedulerClusterCount int64 - if err := db.Model(model.SchedulerCluster{}).Count(&schedulerClusterCount).Error; err != nil { - return err - } - if schedulerClusterCount <= 0 { - if err := db.Create(&model.SchedulerCluster{ - Model: model.Model{ - ID: uint(1), - }, - Name: "scheduler-cluster-1", - Config: map[string]interface{}{}, - ClientConfig: map[string]interface{}{ - "load_limit": defaultClientLoadLimit, - }, - Scopes: map[string]interface{}{}, - IsDefault: true, - }).Error; err != nil { - return err - } - } - - if schedulerClusterCount == 0 && cdnClusterCount == 0 { - cdnCluster := model.CDNCluster{} - if err := db.First(&cdnCluster).Error; err != nil { - return err - } - - schedulerCluster := model.SchedulerCluster{} - if err := db.First(&schedulerCluster).Error; err != nil { - return err - } - - if err := db.Model(&cdnCluster).Association("SchedulerClusters").Append(&schedulerCluster); err != nil { - return err - } - } - - return nil -} diff --git a/manager/database/mysql.go b/manager/database/mysql.go new file mode 100644 index 00000000000..8fb770e69a9 --- /dev/null +++ b/manager/database/mysql.go @@ -0,0 +1,176 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package database + +import ( + "fmt" + "time" + + "github.com/go-sql-driver/mysql" + drivermysql "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/schema" + "moul.io/zapgorm2" + + logger "d7y.io/dragonfly/v2/internal/dflog" + "d7y.io/dragonfly/v2/manager/config" + "d7y.io/dragonfly/v2/manager/model" +) + +const ( + defaultCDNLoadLimit = 300 + defaultClientLoadLimit = 100 +) + +func newMyqsl(cfg *config.MysqlConfig) (*gorm.DB, error) { + // Format dsn string + dsn, err := formatDSN(cfg) + if err != nil { + return nil, err + } + + // Connect to mysql + db, err := gorm.Open(drivermysql.Open(dsn), &gorm.Config{ + NamingStrategy: schema.NamingStrategy{ + SingularTable: true, + }, + DisableForeignKeyConstraintWhenMigrating: true, + Logger: zapgorm2.New(logger.CoreLogger.Desugar()), + }) + if err != nil { + return nil, err + } + + // Run migration + if cfg.Migrate { + if err := migrate(db); err != nil { + return nil, err + } + } + + // Run seed + if err := seed(db); err != nil { + return nil, err + } + + return db, nil +} + +func formatDSN(cfg *config.MysqlConfig) (string, error) { + mysqlCfg := mysql.Config{ + User: cfg.User, + Passwd: cfg.Password, + Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), + Net: "tcp", + DBName: cfg.DBName, + Loc: time.Local, + AllowNativePasswords: true, + ParseTime: true, + InterpolateParams: true, + } + + // Support TLS connection + if cfg.TLS != nil { + mysqlCfg.TLSConfig = "custom" + tls, err := cfg.TLS.Client() + if err != nil { + return "", err + } + + if err := mysql.RegisterTLSConfig("custom", tls); err != nil { + return "", err + } + } + + return mysqlCfg.FormatDSN(), nil +} + +func migrate(db *gorm.DB) error { + return db.Set("gorm:table_options", "DEFAULT CHARSET=utf8mb4 ROW_FORMAT=Dynamic").AutoMigrate( + &model.Job{}, + &model.CDNCluster{}, + &model.CDN{}, + &model.SchedulerCluster{}, + &model.Scheduler{}, + &model.SecurityRule{}, + &model.SecurityGroup{}, + &model.User{}, + &model.Oauth{}, + &model.Config{}, + &model.Application{}, + ) +} + +func seed(db *gorm.DB) error { + var cdnClusterCount int64 + if err := db.Model(model.CDNCluster{}).Count(&cdnClusterCount).Error; err != nil { + return err + } + if cdnClusterCount <= 0 { + if err := db.Create(&model.CDNCluster{ + Model: model.Model{ + ID: uint(1), + }, + Name: "cdn-cluster-1", + Config: map[string]interface{}{ + "load_limit": defaultCDNLoadLimit, + }, + IsDefault: true, + }).Error; err != nil { + return err + } + } + + var schedulerClusterCount int64 + if err := db.Model(model.SchedulerCluster{}).Count(&schedulerClusterCount).Error; err != nil { + return err + } + if schedulerClusterCount <= 0 { + if err := db.Create(&model.SchedulerCluster{ + Model: model.Model{ + ID: uint(1), + }, + Name: "scheduler-cluster-1", + Config: map[string]interface{}{}, + ClientConfig: map[string]interface{}{ + "load_limit": defaultClientLoadLimit, + }, + Scopes: map[string]interface{}{}, + IsDefault: true, + }).Error; err != nil { + return err + } + } + + if schedulerClusterCount == 0 && cdnClusterCount == 0 { + cdnCluster := model.CDNCluster{} + if err := db.First(&cdnCluster).Error; err != nil { + return err + } + + schedulerCluster := model.SchedulerCluster{} + if err := db.First(&schedulerCluster).Error; err != nil { + return err + } + + if err := db.Model(&cdnCluster).Association("SchedulerClusters").Append(&schedulerCluster); err != nil { + return err + } + } + + return nil +} diff --git a/manager/database/redis.go b/manager/database/redis.go new file mode 100644 index 00000000000..d7f842382da --- /dev/null +++ b/manager/database/redis.go @@ -0,0 +1,42 @@ +/* + * Copyright 2020 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package database + +import ( + "context" + "fmt" + + "github.com/go-redis/redis/v8" + + "d7y.io/dragonfly/v2/manager/config" +) + +func NewRedis(cfg *config.RedisConfig) (*redis.Client, error) { + redis.SetLogger(&redisLogger{}) + + client := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), + Password: cfg.Password, + DB: cfg.CacheDB, + }) + + if err := client.Ping(context.Background()).Err(); err != nil { + return nil, err + } + + return client, nil +}