diff --git a/engine/Makefile b/engine/Makefile index a9250148e8..c4cb8145d2 100644 --- a/engine/Makefile +++ b/engine/Makefile @@ -43,6 +43,7 @@ TEST_DB_START_DOCKER = docker run -d -p $(TEST_DB_PORT):5432 -e POSTGRES_PASSWOR TEST_REDIS_PORT = 6379 TEST_REDIS_HOST = $(if ${CDS_CACHE_REDIS_HOST},${CDS_CACHE_REDIS_HOST},localhost:$(TEST_REDIS_PORT)) TEST_REDIS_PASS = $(if ${CDS_CACHE_REDIS_PASS},${CDS_CACHE_REDIS_PASS},) +TEST_REDIS_DBINDEX = 0 TEST_REDIS_START_DOCKER = docker run -d -p $(TEST_REDIS_PORT):6379 --name redis-cds redis:5 TEST_KAFKA_START_DOCKER = docker run -d -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=localhost --env ADVERTISED_PORT=9092 --name kafka-cds spotify/kafka @@ -148,11 +149,11 @@ test-config: ${HOME}/.cds/api.tests.cfg.json ${HOME}/.cds/cdn.tests.cfg.json ${HOME}/.cds/api.tests.cfg.json: @mkdir -p ${HOME}/.cds - @echo "{\"dbDriver\": \"postgres\", \"dbUser\" :\"$(TEST_DB_USER)\", \"dbPassword\" :\"$(TEST_DB_PASS)\" ,\"dbName\": \"$(TEST_DB_NAME)\", \"dbSchema\": \"$(TEST_API_DB_SCHEMA)\", \"dbHost\": \"$(TEST_DB_HOST)\", \"dbPort\" : \"$(TEST_DB_PORT)\", \"sslMode\": \"disable\", \"redisHost\" : \"$(TEST_REDIS_HOST)\", \"redisPassword\" : \"$(TEST_REDIS_PASSWORD)\"}" > ${HOME}/.cds/api.tests.cfg.json + @echo "{\"dbDriver\": \"postgres\", \"dbUser\" :\"$(TEST_DB_USER)\", \"dbPassword\" :\"$(TEST_DB_PASS)\" ,\"dbName\": \"$(TEST_DB_NAME)\", \"dbSchema\": \"$(TEST_API_DB_SCHEMA)\", \"dbHost\": \"$(TEST_DB_HOST)\", \"dbPort\" : \"$(TEST_DB_PORT)\", \"sslMode\": \"disable\", \"redisHost\" : \"$(TEST_REDIS_HOST)\", \"redisPassword\" : \"$(TEST_REDIS_PASSWORD)\", \"redisDbIndex\" : \"$(TEST_REDIS_DBINDEX)\"}" > ${HOME}/.cds/api.tests.cfg.json ${HOME}/.cds/cdn.tests.cfg.json: @mkdir -p ${HOME}/.cds - @echo "{\"dbDriver\": \"postgres\", \"dbUser\" :\"$(TEST_DB_USER)\", \"dbPassword\" :\"$(TEST_DB_PASS)\" ,\"dbName\": \"$(TEST_DB_NAME)\", \"dbSchema\": \"$(TEST_CDN_DB_SCHEMA)\", \"dbHost\": \"$(TEST_DB_HOST)\", \"dbPort\" : \"$(TEST_DB_PORT)\", \"sslMode\": \"disable\", \"redisHost\" : \"$(TEST_REDIS_HOST)\", \"redisPassword\" : \"$(TEST_REDIS_PASSWORD)\"}" > ${HOME}/.cds/cdn.tests.cfg.json + @echo "{\"dbDriver\": \"postgres\", \"dbUser\" :\"$(TEST_DB_USER)\", \"dbPassword\" :\"$(TEST_DB_PASS)\" ,\"dbName\": \"$(TEST_DB_NAME)\", \"dbSchema\": \"$(TEST_CDN_DB_SCHEMA)\", \"dbHost\": \"$(TEST_DB_HOST)\", \"dbPort\" : \"$(TEST_DB_PORT)\", \"sslMode\": \"disable\", \"redisHost\" : \"$(TEST_REDIS_HOST)\", \"redisPassword\" : \"$(TEST_REDIS_PASSWORD)\", \"redisDbIndex\" : \"$(TEST_REDIS_DBINDEX)\"}" > ${HOME}/.cds/cdn.tests.cfg.json # Build a list of the first level of package in the 'engine' package, then remove the '.', 'dist' and 'sql' TEST_COMPONENTS = $(addprefix test-component-,$(filter-out sql,$(filter-out dist,$(filter-out .,$(shell find . -maxdepth 1 -type d -exec basename {} \;))))) diff --git a/engine/api/api.go b/engine/api/api.go index a022654205..13d864d32c 100644 --- a/engine/api/api.go +++ b/engine/api/api.go @@ -74,6 +74,7 @@ type Configuration struct { Redis struct { Host string `toml:"host" default:"localhost:6379" comment:"If your want to use a redis-sentinel based cluster, follow this syntax! @sentinel1:26379,sentinel2:26379,sentinel3:26379" json:"host"` Password string `toml:"password" json:"-"` + DbIndex int `toml:"dbindex" default:"0" json:"dbindex"` } `toml:"redis" comment:"Connect CDS to a redis cache If you more than one CDS instance and to avoid losing data at startup" json:"redis"` } `toml:"cache" comment:"######################\n CDS Cache Settings \n#####################" json:"cache"` Download struct { @@ -535,6 +536,7 @@ func (a *API) Serve(ctx context.Context) error { a.Cache, err = cache.New( a.Config.Cache.Redis.Host, a.Config.Cache.Redis.Password, + a.Config.Cache.Redis.DbIndex, a.Config.Cache.TTL) if err != nil { return sdk.WrapError(err, "cannot connect to cache store") diff --git a/engine/cache/cache.go b/engine/cache/cache.go index b8d80608bf..4acd10703d 100644 --- a/engine/cache/cache.go +++ b/engine/cache/cache.go @@ -96,8 +96,8 @@ type SetValueWithScore struct { } //New init a cache -func New(redisHost, redisPassword string, TTL int) (Store, error) { - return NewRedisStore(redisHost, redisPassword, TTL) +func New(redisHost, redisPassword string, dbindex, TTL int) (Store, error) { + return NewRedisStore(redisHost, redisPassword, dbindex, TTL) } //NewWriteCloser returns a write closer diff --git a/engine/cache/redis.go b/engine/cache/redis.go index 91cf2ce51d..0a824984f4 100644 --- a/engine/cache/redis.go +++ b/engine/cache/redis.go @@ -26,7 +26,7 @@ type RedisStore struct { } //NewRedisStore initiate a new redisStore -func NewRedisStore(host, password string, ttl int) (*RedisStore, error) { +func NewRedisStore(host, password string, dbindex, ttl int) (*RedisStore, error) { var client *redis.Client //if host is line master@localhost:26379,localhost:26380 => it's a redis sentinel cluster @@ -38,6 +38,7 @@ func NewRedisStore(host, password string, ttl int) (*RedisStore, error) { MasterName: masterName, SentinelAddrs: sentinels, Password: password, + DB: dbindex, IdleCheckFrequency: 10 * time.Second, IdleTimeout: 10 * time.Second, PoolSize: 25, @@ -50,7 +51,7 @@ func NewRedisStore(host, password string, ttl int) (*RedisStore, error) { client = redis.NewClient(&redis.Options{ Addr: host, Password: password, // no password set - DB: 0, // use default DB + DB: dbindex, IdleCheckFrequency: 30 * time.Second, MaxRetries: 10, MinRetryBackoff: 30 * time.Millisecond, diff --git a/engine/cache/redis_test.go b/engine/cache/redis_test.go index 5529cfe21f..41f57f64df 100644 --- a/engine/cache/redis_test.go +++ b/engine/cache/redis_test.go @@ -2,6 +2,7 @@ package cache import ( "context" + "strconv" "testing" "time" @@ -17,7 +18,9 @@ func TestSortedSet(t *testing.T) { cfg := testConfig.LoadTestingConf(t, sdk.TypeAPI) redisHost := cfg["redisHost"] redisPassword := cfg["redisPassword"] - s, err := NewRedisStore(redisHost, redisPassword, 60) + redisDbIndex, err := strconv.ParseInt(cfg["redisDbIndex"], 10, 64) + require.NoError(t, err, "error when unmarshal config") + s, err := NewRedisStore(redisHost, redisPassword, int(redisDbIndex), 60) require.NoError(t, err) s.Delete("test") @@ -33,7 +36,9 @@ func TestDequeueJSONRawMessagesWithContext(t *testing.T) { cfg := testConfig.LoadTestingConf(t, sdk.TypeAPI) redisHost := cfg["redisHost"] redisPassword := cfg["redisPassword"] - s, err := NewRedisStore(redisHost, redisPassword, 60) + redisDbIndex, err := strconv.ParseInt(cfg["redisDbIndex"], 10, 64) + require.NoError(t, err, "error when unmarshal config") + s, err := NewRedisStore(redisHost, redisPassword, int(redisDbIndex), 60) require.NoError(t, err) s.Delete("test") @@ -71,7 +76,9 @@ func TestDequeueJSONRawMessagesWithContextMaxTimeout(t *testing.T) { cfg := testConfig.LoadTestingConf(t, sdk.TypeAPI) redisHost := cfg["redisHost"] redisPassword := cfg["redisPassword"] - s, err := NewRedisStore(redisHost, redisPassword, 60) + redisDbIndex, err := strconv.ParseInt(cfg["redisDbIndex"], 10, 64) + require.NoError(t, err, "error when unmarshal config") + s, err := NewRedisStore(redisHost, redisPassword, int(redisDbIndex), 60) require.NoError(t, err) s.Delete("test") diff --git a/engine/cdn/cdn.go b/engine/cdn/cdn.go index ced35f6521..f60debf31a 100644 --- a/engine/cdn/cdn.go +++ b/engine/cdn/cdn.go @@ -115,7 +115,7 @@ func (s *Service) Start(ctx context.Context) error { var err error log.Info(ctx, "Initializing redis cache on %s...", s.Cfg.Cache.Redis.Host) - s.Cache, err = cache.New(s.Cfg.Cache.Redis.Host, s.Cfg.Cache.Redis.Password, s.Cfg.Cache.TTL) + s.Cache, err = cache.New(s.Cfg.Cache.Redis.Host, s.Cfg.Cache.Redis.Password, s.Cfg.Cache.Redis.DbIndex, s.Cfg.Cache.TTL) if err != nil { return sdk.WrapError(err, "cannot connect to redis instance") } @@ -140,7 +140,7 @@ func (s *Service) Start(ctx context.Context) error { storage.InitDBMapping(s.Mapper) log.Info(ctx, "Initializing lru connection...") - s.LogCache, err = lru.NewRedisLRU(s.mustDBWithCtx(ctx), s.Cfg.Cache.LruSize, s.Cfg.Cache.Redis.Host, s.Cfg.Cache.Redis.Password) + s.LogCache, err = lru.NewRedisLRU(s.mustDBWithCtx(ctx), s.Cfg.Cache.LruSize, s.Cfg.Cache.Redis.Host, s.Cfg.Cache.Redis.Password, s.Cfg.Cache.Redis.DbIndex) if err != nil { return sdk.WrapError(err, "cannot connect to redis instance for lru") } diff --git a/engine/cdn/cdn_gc_test.go b/engine/cdn/cdn_gc_test.go index cfa20efe60..c11f2bf872 100644 --- a/engine/cdn/cdn_gc_test.go +++ b/engine/cdn/cdn_gc_test.go @@ -55,6 +55,7 @@ func TestCleanSynchronizedItem(t *testing.T) { Redis: &storage.RedisBufferConfiguration{ Host: cfg["redisHost"], Password: cfg["redisPassword"], + DbIndex: 0, }, BufferType: storage.CDNBufferTypeLog, }, @@ -184,6 +185,7 @@ func TestCleanSynchronizedItemWithDisabledStorage(t *testing.T) { Redis: &storage.RedisBufferConfiguration{ Host: cfg["redisHost"], Password: cfg["redisPassword"], + DbIndex: 0, }, BufferType: storage.CDNBufferTypeLog, }, @@ -385,7 +387,7 @@ func TestPurgeItem(t *testing.T) { var err error cfg := test.LoadTestingConf(t, sdk.TypeCDN) - s.LogCache, err = lru.NewRedisLRU(db.DbMap, 1000, cfg["redisHost"], cfg["redisPassword"]) + s.LogCache, err = lru.NewRedisLRU(db.DbMap, 1000, cfg["redisHost"], cfg["redisPassword"], 0) require.NoError(t, err) // Add Item in CDS and FS diff --git a/engine/cdn/cdn_item_test.go b/engine/cdn/cdn_item_test.go index ca6b013dc8..c89e2d5d84 100644 --- a/engine/cdn/cdn_item_test.go +++ b/engine/cdn/cdn_item_test.go @@ -50,7 +50,7 @@ func TestGetItemValue(t *testing.T) { cdnUnits := newRunningStorageUnits(t, m, s.DBConnectionFactory.GetDBMap(m)(), ctx, cache) s.Units = cdnUnits var err error - s.LogCache, err = lru.NewRedisLRU(db.DbMap, 1000, cfg["redisHost"], cfg["redisPassword"]) + s.LogCache, err = lru.NewRedisLRU(db.DbMap, 1000, cfg["redisHost"], cfg["redisPassword"], 0) require.NoError(t, err) require.NoError(t, s.LogCache.Clear()) @@ -209,7 +209,7 @@ func TestGetItemValue_ThousandLines(t *testing.T) { cdnUnits := newRunningStorageUnits(t, m, db.DbMap, ctx, cache) s.Units = cdnUnits var err error - s.LogCache, err = lru.NewRedisLRU(db.DbMap, 1000, cfg["redisHost"], cfg["redisPassword"]) + s.LogCache, err = lru.NewRedisLRU(db.DbMap, 1000, cfg["redisHost"], cfg["redisPassword"], 0) require.NoError(t, err) require.NoError(t, s.LogCache.Clear()) @@ -316,7 +316,7 @@ func TestGetItemValue_Reverse(t *testing.T) { cdnUnits := newRunningStorageUnits(t, m, db.DbMap, ctx, cache) s.Units = cdnUnits var err error - s.LogCache, err = lru.NewRedisLRU(db.DbMap, 1000, cfg["redisHost"], cfg["redisPassword"]) + s.LogCache, err = lru.NewRedisLRU(db.DbMap, 1000, cfg["redisHost"], cfg["redisPassword"], 0) require.NoError(t, err) require.NoError(t, s.LogCache.Clear()) @@ -426,7 +426,7 @@ func TestGetItemValue_ThousandLinesReverse(t *testing.T) { cdnUnits := newRunningStorageUnits(t, m, db.DbMap, ctx, cache) s.Units = cdnUnits var err error - s.LogCache, err = lru.NewRedisLRU(db.DbMap, 1000, cfg["redisHost"], cfg["redisPassword"]) + s.LogCache, err = lru.NewRedisLRU(db.DbMap, 1000, cfg["redisHost"], cfg["redisPassword"], 0) require.NoError(t, err) require.NoError(t, s.LogCache.Clear()) diff --git a/engine/cdn/cdn_sync_test.go b/engine/cdn/cdn_sync_test.go index 4f3191b10c..4670da89d3 100644 --- a/engine/cdn/cdn_sync_test.go +++ b/engine/cdn/cdn_sync_test.go @@ -46,6 +46,7 @@ func TestSyncBuffer(t *testing.T) { Redis: &storage.RedisBufferConfiguration{ Host: cfg["redisHost"], Password: cfg["redisPassword"], + DbIndex: 0, }, BufferType: storage.CDNBufferTypeLog, }, diff --git a/engine/cdn/cdn_test.go b/engine/cdn/cdn_test.go index 304f8e3109..dda0ce83ca 100644 --- a/engine/cdn/cdn_test.go +++ b/engine/cdn/cdn_test.go @@ -134,6 +134,7 @@ func newRunningStorageUnits(t *testing.T, m *gorpmapper.Mapper, dbMap *gorp.DbMa Redis: &storage.RedisBufferConfiguration{ Host: cfg["redisHost"], Password: cfg["redisPassword"], + DbIndex: 0, }, BufferType: storage.CDNBufferTypeLog, }, diff --git a/engine/cdn/item_upload_test.go b/engine/cdn/item_upload_test.go index fe73b33db4..263bd87ca7 100644 --- a/engine/cdn/item_upload_test.go +++ b/engine/cdn/item_upload_test.go @@ -63,6 +63,7 @@ func TestPostUploadHandler(t *testing.T) { Redis: &storage.RedisBufferConfiguration{ Host: cfg["redisHost"], Password: cfg["redisPassword"], + DbIndex: 0, }, BufferType: storage.CDNBufferTypeLog, }, diff --git a/engine/cdn/lru/redis.go b/engine/cdn/lru/redis.go index afb76a2d85..c676686ad1 100644 --- a/engine/cdn/lru/redis.go +++ b/engine/cdn/lru/redis.go @@ -28,8 +28,8 @@ type Redis struct { } // NewRedisLRU instanciates a new Redis LRU -func NewRedisLRU(db *gorp.DbMap, maxSize int64, host string, password string) (*Redis, error) { - c, err := cache.New(host, password, -1) +func NewRedisLRU(db *gorp.DbMap, maxSize int64, host string, password string, dbindex int) (*Redis, error) { + c, err := cache.New(host, password, dbindex, -1) if err != nil { return nil, err } diff --git a/engine/cdn/lru/redis_test.go b/engine/cdn/lru/redis_test.go index a59f2c2935..0a41a8c4d8 100644 --- a/engine/cdn/lru/redis_test.go +++ b/engine/cdn/lru/redis_test.go @@ -23,7 +23,7 @@ func TestRedisLRU(t *testing.T) { cdntest.ClearItem(t, context.TODO(), m, db) cfg := test.LoadTestingConf(t, sdk.TypeCDN) - r, err := NewRedisLRU(db.DbMap, 100, cfg["redisHost"], cfg["redisPassword"]) + r, err := NewRedisLRU(db.DbMap, 100, cfg["redisHost"], cfg["redisPassword"], 0) require.NoError(t, err) l, _ := r.Len() diff --git a/engine/cdn/redis/reader_test.go b/engine/cdn/redis/reader_test.go index 035b20ba07..47ede98f85 100644 --- a/engine/cdn/redis/reader_test.go +++ b/engine/cdn/redis/reader_test.go @@ -15,7 +15,7 @@ import ( func TestReader_EOF(t *testing.T) { cfg := test.LoadTestingConf(t, sdk.TypeCDN) - c, err := cache.New(cfg["redisHost"], cfg["redisPassword"], -1) + c, err := cache.New(cfg["redisHost"], cfg["redisPassword"], 0, -1) require.NoError(t, err) cacheKey := cache.Key("test:cdn:item") diff --git a/engine/cdn/redis/writer_test.go b/engine/cdn/redis/writer_test.go index 946a990ab2..df848ac706 100644 --- a/engine/cdn/redis/writer_test.go +++ b/engine/cdn/redis/writer_test.go @@ -14,7 +14,7 @@ import ( func TestWriter_Closed(t *testing.T) { cfg := test.LoadTestingConf(t, sdk.TypeCDN) - c, err := cache.New(cfg["redisHost"], cfg["redisPassword"], -1) + c, err := cache.New(cfg["redisHost"], cfg["redisPassword"], 0, -1) require.NoError(t, err) cacheKey := cache.Key("test:cdn:item") diff --git a/engine/cdn/storage/dao_test.go b/engine/cdn/storage/dao_test.go index 29b93a86bb..41210ac02c 100644 --- a/engine/cdn/storage/dao_test.go +++ b/engine/cdn/storage/dao_test.go @@ -55,6 +55,7 @@ func TestLoadAllItemIDUnknownByUnit(t *testing.T) { Redis: &storage.RedisBufferConfiguration{ Host: cfg["redisHost"], Password: cfg["redisPassword"], + DbIndex: 0, }, BufferType: storage.CDNBufferTypeLog, }, diff --git a/engine/cdn/storage/redis/redis.go b/engine/cdn/storage/redis/redis.go index a423a6f369..af4ca22327 100644 --- a/engine/cdn/storage/redis/redis.go +++ b/engine/cdn/storage/redis/redis.go @@ -44,7 +44,7 @@ func (s *Redis) Init(_ context.Context, cfg interface{}, bufferType storage.CDNB } s.config = *config var err error - s.store, err = cache.New(s.config.Host, s.config.Password, 60) + s.store, err = cache.New(s.config.Host, s.config.Password, s.config.DbIndex, 60) if err != nil { return err } diff --git a/engine/cdn/storage/storageunit_test.go b/engine/cdn/storage/storageunit_test.go index eeaccb37df..1bdb471b03 100644 --- a/engine/cdn/storage/storageunit_test.go +++ b/engine/cdn/storage/storageunit_test.go @@ -55,6 +55,7 @@ func TestDeduplicationCrossType(t *testing.T) { Redis: &storage.RedisBufferConfiguration{ Host: cfg["redisHost"], Password: cfg["redisPassword"], + DbIndex: 0, }, BufferType: storage.CDNBufferTypeLog, }, @@ -250,6 +251,7 @@ func TestRun(t *testing.T) { Redis: &storage.RedisBufferConfiguration{ Host: cfg["redisHost"], Password: cfg["redisPassword"], + DbIndex: 0, }, BufferType: storage.CDNBufferTypeLog, }, diff --git a/engine/cdn/storage/types.go b/engine/cdn/storage/types.go index 943819e396..0c1d790e7e 100644 --- a/engine/cdn/storage/types.go +++ b/engine/cdn/storage/types.go @@ -222,6 +222,7 @@ type WebdavStorageConfiguration struct { type RedisBufferConfiguration struct { Host string `toml:"host" comment:"If your want to use a redis-sentinel based cluster, follow this syntax ! @sentinel1:26379,sentinel2:26379sentinel3:26379" json:"host"` Password string `toml:"password" json:"-"` + DbIndex int `toml:"dbindex" default:"0" json:"dbindex"` } type LocalBufferConfiguration struct { diff --git a/engine/cdn/types.go b/engine/cdn/types.go index 020af56177..22acc6bf7f 100644 --- a/engine/cdn/types.go +++ b/engine/cdn/types.go @@ -78,6 +78,7 @@ type Configuration struct { Redis struct { Host string `toml:"host" default:"localhost:6379" comment:"If your want to use a redis-sentinel based cluster, follow this syntax ! @sentinel1:26379,sentinel2:26379sentinel3:26379" json:"host"` Password string `toml:"password" json:"-"` + DbIndex int `toml:"dbindex" default:"0" json:"dbindex"` } `toml:"redis" json:"redis"` } `toml:"cache" comment:"######################\n CDN Cache Settings \n######################" json:"cache"` API service.APIServiceConfiguration `toml:"api" comment:"######################\n CDS API Settings \n######################" json:"api"` diff --git a/engine/cdn/unit_handler_test.go b/engine/cdn/unit_handler_test.go index f1172d9edb..8c44d310a9 100644 --- a/engine/cdn/unit_handler_test.go +++ b/engine/cdn/unit_handler_test.go @@ -132,6 +132,7 @@ func TestPostAdminResyncBackendWithDatabaseHandler(t *testing.T) { Redis: &storage.RedisBufferConfiguration{ Host: cfg["redisHost"], Password: cfg["redisPassword"], + DbIndex: 0, }, BufferType: storage.CDNBufferTypeLog, }, diff --git a/engine/hooks/hooks.go b/engine/hooks/hooks.go index 07a1c12fc9..3623d6162e 100644 --- a/engine/hooks/hooks.go +++ b/engine/hooks/hooks.go @@ -86,7 +86,7 @@ func (s *Service) Serve(c context.Context) error { //Init the cache var errCache error - s.Cache, errCache = cache.New(s.Cfg.Cache.Redis.Host, s.Cfg.Cache.Redis.Password, s.Cfg.Cache.TTL) + s.Cache, errCache = cache.New(s.Cfg.Cache.Redis.Host, s.Cfg.Cache.Redis.Password, s.Cfg.Cache.Redis.DbIndex, s.Cfg.Cache.TTL) if errCache != nil { return fmt.Errorf("Cannot connect to redis instance : %v", errCache) } diff --git a/engine/hooks/test.go b/engine/hooks/test.go index ed58e315eb..9a559b56c0 100644 --- a/engine/hooks/test.go +++ b/engine/hooks/test.go @@ -1,6 +1,7 @@ package hooks import ( + "strconv" "testing" "github.com/golang/mock/gomock" @@ -16,10 +17,14 @@ func setupTestHookService(t *testing.T) (Service, func()) { cfg := test.LoadTestingConf(t, sdk.TypeAPI) redisHost := cfg["redisHost"] redisPassword := cfg["redisPassword"] + redisDbIndex, err := strconv.ParseInt(cfg["redisDbIndex"], 10, 64) + if err != nil { + t.Fatalf("redis configuration db index invalid %v", err) + } s.Cfg.RetryError = 1 - store, err := cache.NewRedisStore(redisHost, redisPassword, 60) + store, err := cache.NewRedisStore(redisHost, redisPassword, int(redisDbIndex), 60) if err != nil { t.Fatalf("Unable to connect to redis: %v", err) } diff --git a/engine/hooks/types.go b/engine/hooks/types.go index c01d22631d..dd2691535c 100644 --- a/engine/hooks/types.go +++ b/engine/hooks/types.go @@ -2,6 +2,7 @@ package hooks import ( "crypto/rsa" + "github.com/ovh/cds/engine/api" "github.com/ovh/cds/engine/cache" "github.com/ovh/cds/engine/service" @@ -42,6 +43,7 @@ type Configuration struct { Redis struct { Host string `toml:"host" default:"localhost:6379" comment:"If your want to use a redis-sentinel based cluster, follow this syntax! @sentinel1:26379,sentinel2:26379,sentinel3:26379" json:"host"` Password string `toml:"password" json:"-"` + DbIndex int `toml:"dbindex" default:"0" json:"dbindex"` } `toml:"redis" comment:"Connect CDS to a redis cache If you more than one CDS instance and to avoid losing data at startup" json:"redis"` } `toml:"cache" comment:"######################\n CDS Hooks Cache Settings \n######################" json:"cache"` WebhooksPublicKeySign string `toml:"webhooksPublicKeySign" comment:"Public key to check call signature on handler /v2/webhook/repository"` diff --git a/engine/repositories/repositories.go b/engine/repositories/repositories.go index 383692110b..a5b03d2064 100644 --- a/engine/repositories/repositories.go +++ b/engine/repositories/repositories.go @@ -83,7 +83,7 @@ func (s *Service) Serve(c context.Context) error { //Init the cache log.Info(ctx, "Initializing Redis connection (%s)...", s.Cfg.Cache.Redis.Host) var errCache error - s.Cache, errCache = cache.New(s.Cfg.Cache.Redis.Host, s.Cfg.Cache.Redis.Password, s.Cfg.Cache.TTL) + s.Cache, errCache = cache.New(s.Cfg.Cache.Redis.Host, s.Cfg.Cache.Redis.Password, s.Cfg.Cache.Redis.DbIndex, s.Cfg.Cache.TTL) if errCache != nil { return fmt.Errorf("cannot connect to redis instance : %v", errCache) } diff --git a/engine/repositories/repositories_test.go b/engine/repositories/repositories_test.go index 2e1a6428ce..0584324fcc 100644 --- a/engine/repositories/repositories_test.go +++ b/engine/repositories/repositories_test.go @@ -44,6 +44,7 @@ func newTestService(t *testing.T) (*Service, error) { cfg.Cache.TTL = 30 cfg.Cache.Redis.Host = RedisHost cfg.Cache.Redis.Password = RedisPassword + cfg.Cache.Redis.DbIndex = 0 ctx := context.Background() r := &api.Router{ @@ -64,7 +65,7 @@ func newTestService(t *testing.T) (*Service, error) { //Init the cache var errCache error - service.Cache, errCache = cache.New(service.Cfg.Cache.Redis.Host, service.Cfg.Cache.Redis.Password, service.Cfg.Cache.TTL) + service.Cache, errCache = cache.New(service.Cfg.Cache.Redis.Host, service.Cfg.Cache.Redis.Password, service.Cfg.Cache.Redis.DbIndex, service.Cfg.Cache.TTL) if errCache != nil { log.Error(ctx, "Unable to init cache (%s): %v", service.Cfg.Cache.Redis.Host, errCache) return nil, errCache diff --git a/engine/repositories/types.go b/engine/repositories/types.go index 823ef795ae..eaf440d254 100644 --- a/engine/repositories/types.go +++ b/engine/repositories/types.go @@ -33,6 +33,7 @@ type Configuration struct { Redis struct { Host string `toml:"host" default:"localhost:6379" comment:"If your want to use a redis-sentinel based cluster, follow this syntax! @sentinel1:26379,sentinel2:26379,sentinel3:26379" json:"host"` Password string `toml:"password" json:"-"` + DbIndex int `toml:"dbindex" default:"0" json:"dbindex"` } `toml:"redis" json:"redis"` } `toml:"cache" comment:"######################\n CDS Repositories Cache Settings \n######################" json:"cache"` } diff --git a/engine/test/database.go b/engine/test/database.go index fbbd7caafa..93436f6885 100644 --- a/engine/test/database.go +++ b/engine/test/database.go @@ -90,6 +90,8 @@ func SetupPGToCancel(t require.TestingT, m *gorpmapper.Mapper, serviceType strin dbSSLMode := cfg["sslMode"] redisHost := cfg["redisHost"] redisPassword := cfg["redisPassword"] + redisDbIndex, err := strconv.ParseInt(cfg["redisDbIndex"], 10, 64) + require.NoError(t, err, "error when unmarshal config") sigKeys := database.RollingKeyConfig{ Cipher: "hmac", @@ -145,7 +147,7 @@ func SetupPGToCancel(t require.TestingT, m *gorpmapper.Mapper, serviceType strin require.NoError(t, f(context.TODO(), sdk.DefaultValues{}, factory.GetDBMap(m))) } - store, err := cache.NewRedisStore(redisHost, redisPassword, 60) + store, err := cache.NewRedisStore(redisHost, redisPassword, int(redisDbIndex), 60) require.NoError(t, err, "unable to connect to redis") cancel := func() { diff --git a/engine/vcs/bitbucketcloud/bitbucketcloud_test.go b/engine/vcs/bitbucketcloud/bitbucketcloud_test.go index 02d0acb491..c2206edcbf 100644 --- a/engine/vcs/bitbucketcloud/bitbucketcloud_test.go +++ b/engine/vcs/bitbucketcloud/bitbucketcloud_test.go @@ -39,7 +39,7 @@ func getNewConsumer(t *testing.T) sdk.VCSServer { t.SkipNow() } - cache, err := cache.New(redisHost, redisPassword, 30) + cache, err := cache.New(redisHost, redisPassword, 0, 30) if err != nil { t.Fatalf("Unable to init cache (%s): %v", redisHost, err) } @@ -61,7 +61,7 @@ func getNewAuthorizedClient(t *testing.T) sdk.VCSAuthorizedClient { t.SkipNow() } - cache, err := cache.New(redisHost, redisPassword, 30) + cache, err := cache.New(redisHost, redisPassword, 0, 30) if err != nil { t.Fatalf("Unable to init cache (%s): %v", redisHost, err) } diff --git a/engine/vcs/bitbucketserver/bitbucketserver_test.go b/engine/vcs/bitbucketserver/bitbucketserver_test.go index d89f81f722..e5151c770f 100644 --- a/engine/vcs/bitbucketserver/bitbucketserver_test.go +++ b/engine/vcs/bitbucketserver/bitbucketserver_test.go @@ -33,7 +33,7 @@ func getNewConsumer(t *testing.T) sdk.VCSServer { t.SkipNow() } - cache, err := cache.New(redisHost, redisPassword, 30) + cache, err := cache.New(redisHost, redisPassword, 0, 30) if err != nil { t.Fatalf("Unable to init cache (%s): %v", redisHost, err) } @@ -60,7 +60,7 @@ func getAuthorizedClient(t *testing.T) sdk.VCSAuthorizedClient { t.SkipNow() } - cache, err := cache.New(redisHost, redisPassword, 30) + cache, err := cache.New(redisHost, redisPassword, 0, 30) if err != nil { t.Fatalf("Unable to init cache (%s): %v", redisHost, err) } diff --git a/engine/vcs/github/github_test.go b/engine/vcs/github/github_test.go index 87c8ca9832..028400c300 100644 --- a/engine/vcs/github/github_test.go +++ b/engine/vcs/github/github_test.go @@ -35,7 +35,7 @@ func getNewConsumer(t *testing.T) sdk.VCSServer { t.SkipNow() } - cache, err := cache.New(redisHost, redisPassword, 30) + cache, err := cache.New(redisHost, redisPassword, 0, 30) if err != nil { t.Fatalf("Unable to init cache (%s): %v", redisHost, err) } @@ -58,7 +58,7 @@ func getNewAuthorizedClient(t *testing.T) sdk.VCSAuthorizedClient { t.SkipNow() } - cache, err := cache.New(redisHost, redisPassword, 30) + cache, err := cache.New(redisHost, redisPassword, 0, 30) if err != nil { t.Fatalf("Unable to init cache (%s): %v", redisHost, err) } diff --git a/engine/vcs/gitlab/gitlab_test.go b/engine/vcs/gitlab/gitlab_test.go index 667293804a..68b3ee6176 100644 --- a/engine/vcs/gitlab/gitlab_test.go +++ b/engine/vcs/gitlab/gitlab_test.go @@ -35,7 +35,7 @@ func getNewConsumer(t *testing.T) sdk.VCSServer { t.SkipNow() } - cache, err := cache.New(redisHost, redisPassword, 30) + cache, err := cache.New(redisHost, redisPassword, 0, 30) if err != nil { t.Fatalf("Unable to init cache (%s): %v", redisHost, err) } @@ -58,7 +58,7 @@ func getNewAuthorizedClient(t *testing.T) sdk.VCSAuthorizedClient { t.SkipNow() } - cache, err := cache.New(redisHost, redisPassword, 30) + cache, err := cache.New(redisHost, redisPassword, 0, 30) if err != nil { t.Fatalf("Unable to init cache (%s): %v", redisHost, err) } diff --git a/engine/vcs/types.go b/engine/vcs/types.go index 799c7de179..04c96cd6f7 100644 --- a/engine/vcs/types.go +++ b/engine/vcs/types.go @@ -34,6 +34,7 @@ type Configuration struct { Redis struct { Host string `toml:"host" default:"localhost:6379" comment:"If your want to use a redis-sentinel based cluster, follow this syntax ! @sentinel1:26379,sentinel2:26379sentinel3:26379" json:"host"` Password string `toml:"password" json:"-"` + DbIndex int `toml:"dbindex" default:"0" json:"dbindex"` } `toml:"redis" json:"redis"` } `toml:"cache" comment:"######################\n CDS VCS Cache Settings \n######################" json:"cache"` ProxyWebhook string `toml:"proxyWebhook" default:"" commented:"true" comment:"If you want to have a reverse proxy url for your repository webhook, for example if you put https://myproxy.com it will generate a webhook URL like this https://myproxy.com/UUID_OF_YOUR_WEBHOOK" json:"proxy_webhook"` diff --git a/engine/vcs/vcs.go b/engine/vcs/vcs.go index f7ebbd0227..135370d273 100644 --- a/engine/vcs/vcs.go +++ b/engine/vcs/vcs.go @@ -226,7 +226,7 @@ func (s *Service) Serve(c context.Context) error { //Init the cache var errCache error - s.Cache, errCache = cache.New(s.Cfg.Cache.Redis.Host, s.Cfg.Cache.Redis.Password, s.Cfg.Cache.TTL) + s.Cache, errCache = cache.New(s.Cfg.Cache.Redis.Host, s.Cfg.Cache.Redis.Password, s.Cfg.Cache.Redis.DbIndex, s.Cfg.Cache.TTL) if errCache != nil { return fmt.Errorf("Cannot connect to redis instance : %v", errCache) } diff --git a/engine/vcs/vcs_test.go b/engine/vcs/vcs_test.go index e373d5b5a6..b594d7a96a 100644 --- a/engine/vcs/vcs_test.go +++ b/engine/vcs/vcs_test.go @@ -48,6 +48,7 @@ func newTestService(t *testing.T) (*Service, error) { cfg.Cache.TTL = 30 cfg.Cache.Redis.Host = RedisHost cfg.Cache.Redis.Password = RedisPassword + cfg.Cache.Redis.DbIndex = 0 ctx := context.Background() r := &api.Router{ @@ -68,7 +69,7 @@ func newTestService(t *testing.T) (*Service, error) { //Init the cache var errCache error - service.Cache, errCache = cache.New(service.Cfg.Cache.Redis.Host, service.Cfg.Cache.Redis.Password, service.Cfg.Cache.TTL) + service.Cache, errCache = cache.New(service.Cfg.Cache.Redis.Host, service.Cfg.Cache.Redis.Password, service.Cfg.Cache.Redis.DbIndex, service.Cfg.Cache.TTL) if errCache != nil { log.Error(ctx, "Unable to init cache (%s): %v", service.Cfg.Cache.Redis.Host, errCache) return nil, errCache