Skip to content

Commit

Permalink
Fixed redis sentinel (#302)
Browse files Browse the repository at this point in the history
  • Loading branch information
KarelCemus committed May 7, 2024
1 parent 57e54e8 commit f67dd0a
Show file tree
Hide file tree
Showing 6 changed files with 127 additions and 39 deletions.
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
version: '3.7'
version: '3.9'

services:

redis-master:
image: redis:latest
image: redis:7.2
hostname: redis-master
ports:
- '${REDIS_MASTER_PORT}:6379'

redis-slave:
image: redis:latest
image: redis:7.2
hostname: redis-slave
ports:
- '${REDIS_SLAVE_PORT}:6379'
Expand Down
79 changes: 79 additions & 0 deletions docker/sentinel/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
version: '3.9'

services:

redis-master:
image: "bitnami/redis:7.2"
hostname: redis-master
ports:
- "${REDIS_MASTER_PORT}:6379"
networks:
- redis-sentinel
environment:
- REDIS_REPLICATION_MODE=master
- ALLOW_EMPTY_PASSWORD=yes

redis-slave:
image: "bitnami/redis:7.2"
hostname: redis-slave
ports:
- "${REDIS_SLAVE_PORT}:6379"
networks:
- redis-sentinel
environment:
- REDIS_REPLICATION_MODE=slave
- REDIS_MASTER_HOST=redis-master
- ALLOW_EMPTY_PASSWORD=yes
depends_on:
- redis-master

redis-sentinel-1:
image: "bitnami/redis-sentinel:7.2"
hostname: redis-sentinel-1
ports:
- "${REDIS_SENTINEL_1_PORT}:26379"
networks:
- redis-sentinel
environment:
- REDIS_MASTER_SET=mymaster
- REDIS_MASTER_HOST=redis-master
- REDIS_SENTINEL_DOWN_AFTER_MILLISECONDS=10000
- ALLOW_EMPTY_PASSWORD=yes
depends_on:
- redis-master
- redis-slave

redis-sentinel-2:
image: "bitnami/redis-sentinel:7.2"
hostname: redis-sentinel-2
ports:
- "${REDIS_SENTINEL_2_PORT}:26379"
networks:
- redis-sentinel
environment:
- REDIS_MASTER_SET=mymaster
- REDIS_MASTER_HOST=redis-master
- REDIS_SENTINEL_DOWN_AFTER_MILLISECONDS=10000
- ALLOW_EMPTY_PASSWORD=yes
depends_on:
- redis-master
- redis-slave

redis-sentinel-3:
image: "bitnami/redis-sentinel:7.2"
hostname: redis-sentinel-3
ports:
- "${REDIS_SENTINEL_3_PORT}:26379"
networks:
- redis-sentinel
environment:
- REDIS_MASTER_SET=mymaster
- REDIS_MASTER_HOST=redis-master
- REDIS_SENTINEL_DOWN_AFTER_MILLISECONDS=10000
- ALLOW_EMPTY_PASSWORD=yes
depends_on:
- redis-master
- redis-slave

networks:
redis-sentinel:
Original file line number Diff line number Diff line change
Expand Up @@ -213,10 +213,10 @@ private[connector] class RedisCommandsSentinel(

private val redisUri: RedisURI =
RedisURI.Builder
.sentinel(sentinel.host, sentinel.port)
.sentinel(sentinel.host, sentinel.port, configuration.masterGroup)
.withDatabase(configuration.database)
.withCredentials(configuration.username, configuration.password)
.withSentinels(configuration.sentinels)
.withSentinels(configuration.sentinels.tail)
.build()

override protected def connectionString: String = redisUri.toString
Expand All @@ -232,7 +232,8 @@ private[connector] class RedisCommandsSentinel(

val newConnection: RedisConnection =
RedisConnection.fromStandalone(
client.connect().withTimeout(configuration.timeout.connection),
MasterReplica.connect(client, StringCodec.UTF8, redisUri)
.withReadFrom(ReadFrom.MASTER_PREFERRED),
)

}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package play.api.cache.redis.connector

import org.apache.pekko.actor.ActorSystem
import org.scalatest.Ignore
import play.api.cache.redis._
import play.api.cache.redis.configuration._
import play.api.cache.redis.impl._
Expand All @@ -11,7 +10,6 @@ import play.api.inject.{ApplicationLifecycle, Injector}
import scala.concurrent.duration._
import scala.concurrent.{ExecutionContext, Future}

@Ignore
class RedisSentinelSpec extends IntegrationSpec with RedisSentinelContainer with DefaultInjector {

test("pong on ping") { connector =>
Expand Down Expand Up @@ -74,10 +72,8 @@ class RedisSentinelSpec extends IntegrationSpec with RedisSentinelContainer with
name = "sentinel",
masterGroup = master,
sentinels = 0
.until(nodes)
.map { i =>
RedisHost(container.containerIpAddress, container.mappedPort(sentinelPort + i))
}
.until(sentinels)
.map(i => RedisHost(host, sentinelPort + i))
.toList,
settings = RedisSettings.load(
config = Helpers.configuration.default.underlying,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ trait RedisMasterSlaveContainer extends ForAllTestContainer {

protected def newContainer: TestContainerDef[TestContainer] =
DockerComposeContainer.Def(
new File("src/test/resources/docker-compose.yml"),
new File("docker/master-slave/docker-compose.yml"),
tailChildContainers = true,
env = Map(
"REDIS_MASTER_PORT" -> s"$masterPort",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,43 +1,53 @@
package play.api.cache.redis.test

import com.dimafeng.testcontainers.{DockerComposeContainer, ExposedService}
import org.scalatest.Suite
import play.api.Logger
import org.testcontainers.containers.wait.strategy.Wait

import scala.concurrent.duration.DurationInt
import java.io.File

trait RedisSentinelContainer extends RedisContainer {
trait RedisSentinelContainer extends ForAllTestContainer {
this: Suite =>

private val log = Logger("play.api.cache.redis.test")
override protected type TestContainer = DockerComposeContainer

protected def nodes: Int = 3
protected def master: String = "mymaster"

final protected def initialPort: Int = 7000
final protected val host = "localhost"

final protected def sentinelPort: Int = initialPort - 2000
final private val initialPort = 7000

final protected val masterPort = initialPort
final protected val slavePort = masterPort + 1

protected def master: String = s"sentinel$initialPort"
protected def sentinels: Int = 3

private val waitForStart = 7.seconds
final protected def sentinelPort: Int = initialPort - 2000

override protected lazy val redisConfig: RedisContainerConfig =
RedisContainerConfig(
redisDockerImage = "grokzen/redis-cluster:7.0.10",
redisMappedPorts = Seq.empty,
redisFixedPorts = 0.until(nodes).flatMap(i => Seq(initialPort + i, sentinelPort + i)),
redisEnvironment = Map(
"IP" -> "0.0.0.0",
"INITIAL_PORT" -> initialPort.toString,
"SENTINEL" -> "true",
// note: waiting for sentinels is not working
// private def sentinelWaitStrategy = new WaitAllStrategy()
// .withStrategy(Wait.forLogMessage(".*\\+monitor master.*\\n", 1))
// .withStrategy(Wait.forLogMessage(".*\\+slave slave.*\\n", 1))

protected def newContainer: TestContainerDef[TestContainer] =
DockerComposeContainer.Def(
new File("docker/sentinel/docker-compose.yml"),
tailChildContainers = true,
env = Map(
"REDIS_MASTER_PORT" -> s"$masterPort",
"REDIS_SLAVE_PORT" -> s"$slavePort",
"REDIS_SENTINEL_1_PORT" -> s"${sentinelPort + 0}",
"REDIS_SENTINEL_2_PORT" -> s"${sentinelPort + 1}",
"REDIS_SENTINEL_3_PORT" -> s"${sentinelPort + 2}",
),
exposedServices = Seq(
ExposedService("redis-master", masterPort, Wait.forLogMessage(".*Ready to accept connections tcp.*\\n", 1)),
ExposedService("redis-slave", slavePort, Wait.forLogMessage(".*MASTER <-> REPLICA sync: Finished with success.*\\n", 1)),
// note: waiting for sentinels doesn't work, it says "service is not running"
// ExposedService("redis-sentinel-1", sentinelPort + 0, sentinelWaitStrategy),
// ExposedService("redis-sentinel-2", sentinelPort + 1, sentinelWaitStrategy),
// ExposedService("redis-sentinel-3", sentinelPort + 2, sentinelWaitStrategy),
),
)

@SuppressWarnings(Array("org.wartremover.warts.ThreadSleep"))
override def beforeAll(): Unit = {
super.beforeAll()
log.info(s"Waiting for Redis Sentinel to start on ${container.containerIpAddress}, will wait for $waitForStart")
Thread.sleep(waitForStart.toMillis)
log.info(s"Finished waiting for Redis Sentinel to start on ${container.containerIpAddress}")
}

}

0 comments on commit f67dd0a

Please sign in to comment.