Skip to content

Commit

Permalink
add cache_table_prefix and data_table_prefix option (#593)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhenghaoz
zhenghaoz committed Nov 18, 2022
1 parent aee5a3d commit 845931f
Show file tree
Hide file tree
Showing 6 changed files with 65 additions and 20 deletions.
18 changes: 15 additions & 3 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,11 @@ type Config struct {

// DatabaseConfig is the configuration for the database.
type DatabaseConfig struct {
DataStore string `mapstructure:"data_store" validate:"required,data_store"` // database for data store
CacheStore string `mapstructure:"cache_store" validate:"required,cache_store"` // database for cache store
TablePrefix string `mapstructure:"table_prefix"`
DataStore string `mapstructure:"data_store" validate:"required,data_store"` // database for data store
CacheStore string `mapstructure:"cache_store" validate:"required,cache_store"` // database for cache store
TablePrefix string `mapstructure:"table_prefix"`
DataTablePrefix string `mapstructure:"data_table_prefix"`
CacheTablePrefix string `mapstructure:"cache_table_prefix"`
}

// MasterConfig is the configuration for the master.
Expand Down Expand Up @@ -526,6 +528,8 @@ func LoadConfig(path string, oneModel bool) (*Config, error) {
{"database.cache_store", "GORSE_CACHE_STORE"},
{"database.data_store", "GORSE_DATA_STORE"},
{"database.table_prefix", "GORSE_TABLE_PREFIX"},
{"database.cache_table_prefix", "GORSE_CACHE_TABLE_PREFIX"},
{"database.data_table_prefix", "GORSE_DATA_TABLE_PREFIX"},
{"master.port", "GORSE_MASTER_PORT"},
{"master.host", "GORSE_MASTER_HOST"},
{"master.http_port", "GORSE_MASTER_HTTP_PORT"},
Expand Down Expand Up @@ -560,6 +564,14 @@ func LoadConfig(path string, oneModel bool) (*Config, error) {
if err := conf.Validate(oneModel); err != nil {
return nil, errors.Trace(err)
}

// apply table prefix
if conf.Database.CacheTablePrefix == "" {
conf.Database.CacheTablePrefix = conf.Database.TablePrefix
}
if conf.Database.DataTablePrefix == "" {
conf.Database.DataTablePrefix = conf.Database.TablePrefix
}
return &conf, nil
}

Expand Down
8 changes: 7 additions & 1 deletion config/config.toml.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,15 @@ cache_store = "redis://localhost:6379/0"
# mongodb+srv://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]]
data_store = "mysql://gorse:gorse_pass@tcp(localhost:3306)/gorse"

# The naming prefix for tables (collections, keys) in databases.
# The naming prefix for tables (collections, keys) in databases. The default value is empty.
table_prefix = ""

# The naming prefix for tables (collections, keys) in cache storage databases. The default value is `table_prefix`.
cache_table_prefix = ""

# The naming prefix for tables (collections, keys) in data storage databases. The default value is `table_prefix`.
data_table_prefix = ""

[master]

# GRPC port of the master node. The default value is 8086.
Expand Down
31 changes: 29 additions & 2 deletions config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package config

import (
"os"
"path/filepath"
"strings"
"testing"
"time"
Expand All @@ -33,6 +34,8 @@ func TestUnmarshal(t *testing.T) {
text = strings.Replace(text, "admin_api_key = \"\"", "admin_api_key = \"super_api_key\"", -1)
text = strings.Replace(text, "api_key = \"\"", "api_key = \"19260817\"", -1)
text = strings.Replace(text, "table_prefix = \"\"", "table_prefix = \"gorse_\"", -1)
text = strings.Replace(text, "cache_table_prefix = \"gorse_\"", "cache_table_prefix = \"gorse_cache_\"", -1)
text = strings.Replace(text, "data_table_prefix = \"gorse_\"", "data_table_prefix = \"gorse_data_\"", -1)
text = strings.Replace(text, "http_cors_domains = []", "http_cors_domains = [\".*\"]", -1)
text = strings.Replace(text, "http_cors_methods = []", "http_cors_methods = [\"GET\",\"PATCH\",\"POST\"]", -1)
viper.SetConfigType("toml")
Expand All @@ -46,6 +49,9 @@ func TestUnmarshal(t *testing.T) {
assert.Equal(t, "redis://localhost:6379/0", config.Database.CacheStore)
assert.Equal(t, "mysql://gorse:gorse_pass@tcp(localhost:3306)/gorse", config.Database.DataStore)
assert.Equal(t, "gorse_", config.Database.TablePrefix)
assert.Equal(t, "gorse_cache_", config.Database.CacheTablePrefix)
assert.Equal(t, "gorse_data_", config.Database.DataTablePrefix)

// [master]
assert.Equal(t, 8086, config.Master.Port)
assert.Equal(t, "0.0.0.0", config.Master.Host)
Expand Down Expand Up @@ -147,6 +153,8 @@ func TestBindEnv(t *testing.T) {
{"GORSE_CACHE_STORE", "redis://<cache_store>"},
{"GORSE_DATA_STORE", "mysql://<data_store>"},
{"GORSE_TABLE_PREFIX", "gorse_"},
{"GORSE_DATA_TABLE_PREFIX", "gorse_data_"},
{"GORSE_CACHE_TABLE_PREFIX", "gorse_cache_"},
{"GORSE_MASTER_PORT", "123"},
{"GORSE_MASTER_HOST", "<master_host>"},
{"GORSE_MASTER_HTTP_PORT", "456"},
Expand All @@ -158,15 +166,16 @@ func TestBindEnv(t *testing.T) {
{"GORSE_SERVER_API_KEY", "<server_api_key>"},
}
for _, variable := range variables {
err := os.Setenv(variable.key, variable.value)
assert.NoError(t, err)
t.Setenv(variable.key, variable.value)
}

config, err := LoadConfig("config.toml.template", false)
assert.NoError(t, err)
assert.Equal(t, "redis://<cache_store>", config.Database.CacheStore)
assert.Equal(t, "mysql://<data_store>", config.Database.DataStore)
assert.Equal(t, "gorse_", config.Database.TablePrefix)
assert.Equal(t, "gorse_cache_", config.Database.CacheTablePrefix)
assert.Equal(t, "gorse_data_", config.Database.DataTablePrefix)
assert.Equal(t, 123, config.Master.Port)
assert.Equal(t, "<master_host>", config.Master.Host)
assert.Equal(t, 456, config.Master.HttpPort)
Expand All @@ -181,6 +190,24 @@ func TestBindEnv(t *testing.T) {
assert.Equal(t, 100, config.Recommend.CacheSize)
}

func TestTablePrefixCompat(t *testing.T) {
data, err := os.ReadFile("config.toml.template")
assert.NoError(t, err)
text := string(data)
text = strings.Replace(text, "cache_table_prefix = \"\"", "", -1)
text = strings.Replace(text, "data_table_prefix = \"\"", "", -1)
text = strings.Replace(text, "table_prefix = \"\"", "table_prefix = \"gorse_\"", -1)
path := filepath.Join(t.TempDir(), "config.toml")
err = os.WriteFile(path, []byte(text), os.ModePerm)
assert.NoError(t, err)

config, err := LoadConfig(path, false)
assert.NoError(t, err)
assert.Equal(t, "gorse_", config.Database.TablePrefix)
assert.Equal(t, "gorse_", config.Database.CacheTablePrefix)
assert.Equal(t, "gorse_", config.Database.DataTablePrefix)
}

func TestConfig_UserNeighborDigest(t *testing.T) {
cfg1, cfg2 := GetDefaultConfig(), GetDefaultConfig()
cfg1.Recommend.UserNeighbors.NeighborType = "auto"
Expand Down
4 changes: 2 additions & 2 deletions master/master.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ func (m *Master) Serve() {
}

// connect data database
m.DataClient, err = data.Open(m.Config.Database.DataStore, m.Config.Database.TablePrefix)
m.DataClient, err = data.Open(m.Config.Database.DataStore, m.Config.Database.DataTablePrefix)
if err != nil {
log.Logger().Fatal("failed to connect data database", zap.Error(err),
zap.String("database", log.RedactDBURL(m.Config.Database.DataStore)))
Expand All @@ -221,7 +221,7 @@ func (m *Master) Serve() {
}

// connect cache database
m.CacheClient, err = cache.Open(m.Config.Database.CacheStore, m.Config.Database.TablePrefix)
m.CacheClient, err = cache.Open(m.Config.Database.CacheStore, m.Config.Database.CacheTablePrefix)
if err != nil {
log.Logger().Fatal("failed to connect cache database", zap.Error(err),
zap.String("database", log.RedactDBURL(m.Config.Database.CacheStore)))
Expand Down
12 changes: 6 additions & 6 deletions server/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,27 +147,27 @@ func (s *Server) Sync() {
}

// connect to data store
if s.dataPath != s.Config.Database.DataStore || s.dataPrefix != s.Config.Database.TablePrefix {
if s.dataPath != s.Config.Database.DataStore || s.dataPrefix != s.Config.Database.DataTablePrefix {
log.Logger().Info("connect data store",
zap.String("database", log.RedactDBURL(s.Config.Database.DataStore)))
if s.DataClient, err = data.Open(s.Config.Database.DataStore, s.Config.Database.TablePrefix); err != nil {
if s.DataClient, err = data.Open(s.Config.Database.DataStore, s.Config.Database.DataTablePrefix); err != nil {
log.Logger().Error("failed to connect data store", zap.Error(err))
goto sleep
}
s.dataPath = s.Config.Database.DataStore
s.dataPrefix = s.Config.Database.TablePrefix
s.dataPrefix = s.Config.Database.DataTablePrefix
}

// connect to cache store
if s.cachePath != s.Config.Database.CacheStore || s.cachePrefix != s.Config.Database.TablePrefix {
if s.cachePath != s.Config.Database.CacheStore || s.cachePrefix != s.Config.Database.CacheTablePrefix {
log.Logger().Info("connect cache store",
zap.String("database", log.RedactDBURL(s.Config.Database.CacheStore)))
if s.CacheClient, err = cache.Open(s.Config.Database.CacheStore, s.Config.Database.TablePrefix); err != nil {
if s.CacheClient, err = cache.Open(s.Config.Database.CacheStore, s.Config.Database.CacheTablePrefix); err != nil {
log.Logger().Error("failed to connect cache store", zap.Error(err))
goto sleep
}
s.cachePath = s.Config.Database.CacheStore
s.cachePrefix = s.Config.Database.TablePrefix
s.cachePrefix = s.Config.Database.CacheTablePrefix
}

// create trace provider
Expand Down
12 changes: 6 additions & 6 deletions worker/worker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,27 +167,27 @@ func (w *Worker) Sync() {
}

// connect to data store
if w.dataPath != w.Config.Database.DataStore || w.dataPrefix != w.Config.Database.TablePrefix {
if w.dataPath != w.Config.Database.DataStore || w.dataPrefix != w.Config.Database.DataTablePrefix {
log.Logger().Info("connect data store",
zap.String("database", log.RedactDBURL(w.Config.Database.DataStore)))
if w.DataClient, err = data.Open(w.Config.Database.DataStore, w.Config.Database.TablePrefix); err != nil {
if w.DataClient, err = data.Open(w.Config.Database.DataStore, w.Config.Database.DataTablePrefix); err != nil {
log.Logger().Error("failed to connect data store", zap.Error(err))
goto sleep
}
w.dataPath = w.Config.Database.DataStore
w.dataPrefix = w.Config.Database.TablePrefix
w.dataPrefix = w.Config.Database.DataTablePrefix
}

// connect to cache store
if w.cachePath != w.Config.Database.CacheStore || w.cachePrefix != w.Config.Database.TablePrefix {
if w.cachePath != w.Config.Database.CacheStore || w.cachePrefix != w.Config.Database.CacheTablePrefix {
log.Logger().Info("connect cache store",
zap.String("database", log.RedactDBURL(w.Config.Database.CacheStore)))
if w.CacheClient, err = cache.Open(w.Config.Database.CacheStore, w.Config.Database.TablePrefix); err != nil {
if w.CacheClient, err = cache.Open(w.Config.Database.CacheStore, w.Config.Database.CacheTablePrefix); err != nil {
log.Logger().Error("failed to connect cache store", zap.Error(err))
goto sleep
}
w.cachePath = w.Config.Database.CacheStore
w.cachePrefix = w.Config.Database.TablePrefix
w.cachePrefix = w.Config.Database.CacheTablePrefix
}

// check ranking model version
Expand Down

0 comments on commit 845931f

Please sign in to comment.