added sentinel support

Makefile

@@ -45,9 +45,13 @@ sequential-unit:
 
 kill_redis:
 	@-redis-cli -p 6668 -a hey_you shutdown
+	@-redis-cli -p 26379 -a hey_you shutdown
+	@-rm /tmp/redis-sentinel.conf 2>/dev/null
 
 redis: kill_redis
+	@cp redis-sentinel.conf /tmp/redis-sentinel.conf
 	@redis-server redis.conf ; sleep 1
+	@redis-server /tmp/redis-sentinel.conf --sentinel ; sleep 1
 	@redis-cli -p 6668 -a hey_you info
 
 format:

docs/configuration.rst

@@ -372,8 +372,17 @@ You have to evaluate the majority of your use cases to take a decision about the
 
    AUTO_PNG_TO_JPG = True
 
-Queueing - Redis
-----------------
+Queueing - Redis Single Node
+----------------------------
+
+REDIS\_QUEUE\_MODE
+~~~~~~~~~~~~~~~~~~
+
+Redis operation mode 'single_node' or 'sentinel'
+
+.. code:: python
+
+   REDIS_QUEUE_MODE = 'single_node'
 
 REDIS\_QUEUE\_SERVER\_HOST
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -411,6 +420,72 @@ Server password for the queued redis detector
 
    REDIS_QUEUE_SERVER_PASSWORD = None
 
+Queueing - Redis Sentinel
+-------------------------
+
+REDIS\_QUEUE\_MODE
+~~~~~~~~~~~~~~~~~~
+
+Redis operation mode 'single_node' or 'sentinel'
+
+.. code:: python
+
+   REDIS_QUEUE_MODE = 'sentinel'
+
+REDIS\_QUEUE\_SENTINEL\_INSTANCES
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sentinel server instances for the queued redis detector.
+
+.. code:: python
+
+   REDIS_QUEUE_SENTINEL_INSTANCES = 'localhost:23679,localhost:23680'
+
+REDIS\_QUEUE\_SENTINEL\_PASSWORD
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sentinel server password for the queued redis detector.
+
+.. code:: python
+
+   REDIS_QUEUE_SENTINEL_PASSWORD = None
+
+REDIS\_QUEUE\_SENTINEL\_MASTER\_INSTANCE
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sentinel server master instance for the queued redis detector.
+
+.. code:: python
+
+   REDIS_QUEUE_SENTINEL_MASTER_INSTANCE = 'masterinstance'
+
+REDIS\_QUEUE\_SENTINEL\_MASTER\_PASSWORD
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sentinel server master password for the queued redis detector.
+
+.. code:: python
+
+   REDIS_QUEUE_SENTINEL_MASTER_PASSWORD = None
+
+REDIS\_QUEUE\_SENTINEL\_MASTER\_DB
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sentinel server master database index for the queued redis detector.
+
+.. code:: python
+
+   REDIS_QUEUE_SENTINEL_MASTER_DB = 0
+
+REDIS\_QUEUE\_SENTINEL\_SOCKET\_TIMEOUT
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sentinel server socket timeout for the queued redis detector.
+
+.. code:: python
+
+   REDIS_QUEUE_SENTINEL_SOCKET_TIMEOUT = 10.0
+
 Queueing - Amazon SQS
 ---------------------
 

docs/lazy_detection.rst

@@ -54,3 +54,9 @@ If the image still hasn't been processed, the same process from before
 applies, except thumbor won't place another message in the queue. This
 is intended as a way not to flood the queue with requests for the same
 image.
+
+Redis Support
+-------------
+
+Thumbor supports `Redis single node <https://redis.io/docs/getting-started/>`_.
+and `Redis sentinel <https://redis.io/docs/manual/sentinel/>`_.

redis-sentinel.conf

@@ -0,0 +1,8 @@
+daemonize yes
+pidfile "/tmp/redis-sentinel.pid"
+port 26379
+requirepass hey_you
+
+sentinel monitor masterinstance 127.0.0.1 6668 2
+sentinel down-after-milliseconds masterinstance 1000
+sentinel auth-pass masterinstance hey_you

setup.py

@@ -32,7 +32,7 @@
     "pytest-cov==3.*,>=3.0.0",
     "pytest-tldr==0.*,>=0.2.1",
     "pytest-xdist==2.*,>=2.4.0",
-    "redis==3.*,>=3.4.0",
+    "redis==4.*,>=4.2.2",
     "remotecv>=2.3.0",
     "sentry-sdk==0.*,>=0.14.1",
     "yanc==0.*,>=0.3.3",

tests/detectors/test_queued_detector.py

@@ -12,7 +12,7 @@
 from unittest import mock
 
 from preggy import expect
-from redis import Redis
+from redis import Redis, Sentinel
 from tornado.testing import gen_test
 
 from tests.base import DetectorTestCase
@@ -22,9 +22,62 @@
 TEST_REDIS_HOST = "0.0.0.0"
 TEST_REDIS_PORT = 6668
 TEST_REDIS_PASSWORD = "hey_you"
+TEST_REDIS_MODE = "single_node"
+TEST_REDIS_SENTINEL_MODE = "sentinel"
+TEST_REDIS_QUEUE_SENTINEL_INSTANCES = "localhost:26379"
+TEST_REDIS_SENTINEL_MASTER_INSTANCE = "masterinstance"
+TEST_REDIS_SENTINEL_SOCKET_TIMEOUT = 10.0
 
 
-class QueuedDetectorTestCase(DetectorTestCase):
+class SharedQueuedDetectorTestCase(DetectorTestCase):
+    def setUp(self):
+        super().setUp()
+        self.redis = None
+
+    async def detector_send_to_queues(self, ctx):
+        detector = QueuedDetector(ctx, 1, [])
+        expect(detector).not_to_be_null()
+
+        data = await detector.detect()
+        expect(data).to_be_empty()
+        expect(ctx.request.detection_error).to_be_false()
+
+        result = self.redis.get("resque:unique:queue:Detect:/image/test.jpg")
+        expect(result).to_equal("1")
+
+        expected_payload = {
+            "queue": "Detect",
+            "args": ["all", "/image/test.jpg", "/image/test.jpg"],
+            "class": "remotecv.pyres_tasks.DetectTask",
+            "key": "/image/test.jpg",
+        }
+
+        result = self.redis.lpop("resque:queue:Detect")
+        expect(loads(result.decode("utf-8"))).to_be_like(expected_payload)
+
+    async def detector_fails_properly(self, ctx):
+        detector = QueuedDetector(ctx, 1, [])
+        expect(detector).not_to_be_null()
+
+        data = await detector.detect()
+        expect(data).to_be_empty()
+        expect(ctx.request.detection_error).to_be_true()
+        expect(detector.queue).to_be_null()
+
+    async def detector_can_detect_twice(self, ctx):
+        detector = QueuedDetector(ctx, 1, [])
+        expect(detector).not_to_be_null()
+
+        data = await detector.detect()
+        expect(data).to_be_empty()
+        expect(ctx.request.detection_error).to_be_false()
+        expect(detector.queue).not_to_be_null()
+
+        data = detector.detect()
+        expect(detector.queue).not_to_be_null()
+
+
+class QueuedDetectorTestCase(SharedQueuedDetectorTestCase):
     def get_config(self):
         return Config(
             REDIS_QUEUE_SERVER_PORT=TEST_REDIS_PORT,
@@ -48,6 +101,7 @@ def setUp(self):
     async def test_detector_sends_to_queue(self):
         ctx = mock.Mock(
             config=mock.Mock(
+                REDIS_QUEUE_MODE=TEST_REDIS_MODE,
                 REDIS_QUEUE_SERVER_HOST=TEST_REDIS_HOST,
                 REDIS_QUEUE_SERVER_PORT=TEST_REDIS_PORT,
                 REDIS_QUEUE_SERVER_DB=0,
@@ -58,31 +112,13 @@ async def test_detector_sends_to_queue(self):
                 detection_error=False,
             ),
         )
-
-        detector = QueuedDetector(ctx, 1, [])
-        expect(detector).not_to_be_null()
-
-        data = await detector.detect()
-        expect(data).to_be_empty()
-        expect(ctx.request.detection_error).to_be_false()
-
-        result = self.redis.get("resque:unique:queue:Detect:/image/test.jpg")
-        expect(result).to_equal("1")
-
-        expected_payload = {
-            "queue": "Detect",
-            "args": ["all", "/image/test.jpg", "/image/test.jpg"],
-            "class": "remotecv.pyres_tasks.DetectTask",
-            "key": "/image/test.jpg",
-        }
-
-        result = self.redis.lpop("resque:queue:Detect")
-        expect(loads(result.decode("utf-8"))).to_be_like(expected_payload)
+        await self.detector_send_to_queues(ctx)
 
     @gen_test
     async def test_detector_fails_properly(self):
         ctx = mock.Mock(
             config=mock.Mock(
+                REDIS_QUEUE_MODE=TEST_REDIS_MODE,
                 REDIS_QUEUE_SERVER_HOST=TEST_REDIS_HOST,
                 REDIS_QUEUE_SERVER_PORT=6669,
                 REDIS_QUEUE_SERVER_DB=0,
@@ -93,19 +129,13 @@ async def test_detector_fails_properly(self):
                 detection_error=False,
             ),
         )
-
-        detector = QueuedDetector(ctx, 1, [])
-        expect(detector).not_to_be_null()
-
-        data = await detector.detect()
-        expect(data).to_be_empty()
-        expect(ctx.request.detection_error).to_be_true()
-        expect(detector.queue).to_be_null()
+        await self.detector_fails_properly(ctx)
 
     @gen_test
     async def test_detector_can_detect_twice(self):
         ctx = mock.Mock(
             config=mock.Mock(
+                REDIS_QUEUE_MODE=TEST_REDIS_MODE,
                 REDIS_QUEUE_SERVER_HOST=TEST_REDIS_HOST,
                 REDIS_QUEUE_SERVER_PORT=TEST_REDIS_PORT,
                 REDIS_QUEUE_SERVER_DB=0,
@@ -116,14 +146,55 @@ async def test_detector_can_detect_twice(self):
                 detection_error=False,
             ),
         )
+        await self.detector_can_detect_twice(ctx)
 
-        detector = QueuedDetector(ctx, 1, [])
-        expect(detector).not_to_be_null()
 
-        data = await detector.detect()
-        expect(data).to_be_empty()
-        expect(ctx.request.detection_error).to_be_false()
-        expect(detector.queue).not_to_be_null()
+class QueuedSentinelDetectorTestCase(SharedQueuedDetectorTestCase):
+    def get_config(self):
+        return Config(
+            REDIS_QUEUE_MODE=TEST_REDIS_SENTINEL_MODE,
+            REDIS_QUEUE_SENTINEL_PASSWORD=TEST_REDIS_PASSWORD,
+            REDIS_QUEUE_SENTINEL_SOCKET_TIMEOUT=TEST_REDIS_SENTINEL_SOCKET_TIMEOUT,
+            REDIS_QUEUE_SENTINEL_MASTER_INSTANCE=TEST_REDIS_SENTINEL_MASTER_INSTANCE,
+            REDIS_QUEUE_SENTINEL_MASTER_PASSWORD=TEST_REDIS_PASSWORD,
+            REDIS_QUEUE_SENTINEL_MASTER_DB=0,
+        )
 
-        data = detector.detect()
-        expect(detector.queue).not_to_be_null()
+    def setUp(self):
+        super().setUp()
+        self.sentinel = Sentinel(
+            [("localhost", 26379)],
+            sentinel_kwargs={"password": TEST_REDIS_PASSWORD},
+        )
+        self.redis = self.sentinel.master_for(
+            TEST_REDIS_SENTINEL_MASTER_INSTANCE,
+            password=TEST_REDIS_PASSWORD,
+            db=0,
+        )
+
+        self.redis.delete("resque:unique:queue:Detect:/image/test.jpg")
+        self.redis.delete("resque:queue:Detect")
+        QueuedDetector.queue = None
+
+        self.request = mock.Mock(
+            image_url="/image/test.jpg",
+            detection_error=False,
+        )
+
+        self.ctx = mock.Mock(
+            config=self.config,
+            request=self.request,
+        )
+
+    @gen_test
+    async def test_detector_sends_to_queue(self):
+        await self.detector_send_to_queues(self.ctx)
+
+    @gen_test
+    async def test_detector_fails_properly(self):
+        self.ctx.config.REDIS_QUEUE_SENTINEL_INSTANCES = "localhost:23680"
+        await self.detector_fails_properly(self.ctx)
+
+    @gen_test
+    async def test_detector_can_detect_twice(self):
+        await self.detector_can_detect_twice(self.ctx)

thumbor/config.py

@@ -672,6 +672,50 @@
     "Server password for the queued redis detector",
     "Queued Redis Detector",
 )
+Config.define(
+    "REDIS_QUEUE_MODE",
+    "single_node",
+    "Redis operation mode 'single_node' or 'sentinel'",
+    "Queued Redis Detector",
+)
+
+# QUEUED DETECTOR REDIS SENTINEL OPTIONS
+Config.define(
+    "REDIS_QUEUE_SENTINEL_INSTANCES",
+    "localhost:26379",
+    "Sentinel server instances for the queued redis detector",
+    "Queued Redis Detector",
+)
+Config.define(
+    "REDIS_QUEUE_SENTINEL_PASSWORD",
+    None,
+    "Sentinel server password for the queued redis detector",
+    "Queued Redis Detector",
+)
+Config.define(
+    "REDIS_QUEUE_SENTINEL_MASTER_INSTANCE",
+    "master",
+    "Sentinel server master instance for the queued redis detector",
+    "Queued Redis Detector",
+)
+Config.define(
+    "REDIS_QUEUE_SENTINEL_MASTER_PASSWORD",
+    None,
+    "Sentinel server master password for the queued redis detector",
+    "Queued Redis Detector",
+)
+Config.define(
+    "REDIS_QUEUE_SENTINEL_MASTER_DB",
+    0,
+    "Sentinel server master database index for the queued redis detector",
+    "Queued Redis Detector",
+)
+Config.define(
+    "REDIS_QUEUE_SENTINEL_SOCKET_TIMEOUT",
+    10.0,
+    "Sentinel server socket timeout for the queued redis detector",
+    "Queued Redis Detector",
+)
 
 # QUEUED DETECTOR SQS OPTIONS
 Config.define("SQS_QUEUE_KEY_ID", None, "AWS key id", "Queued SQS Detector")