Add Thumbor redis storage (#54)

* Fix Redis storage tests

* Address @scorphus comments

Makefile

@@ -9,7 +9,7 @@ run:
 test: run-redis unit stop-redis
 
 unit:
-	@pytest --cov=remotecv --cov-report term-missing --asyncio-mode=strict -r tests/
+	@pytest --cov=remotecv --cov-report term-missing --asyncio-mode=strict -s -r tests/
 
 coverage:
 	@coverage report -m --fail-under=52

remotecv/storages/redis_storage.py

@@ -0,0 +1,82 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# thumbor imaging service
+# https://github.com/thumbor/thumbor/wiki
+
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+# Copyright (c) 2011 globo.com thumbor@googlegroups.com
+
+
+from json import loads
+
+from redis.asyncio import Redis
+from redis.asyncio.sentinel import Sentinel
+from redis import RedisError
+
+
+SINGLE_NODE = "single_node"
+SENTINEL = "sentinel"
+
+
+class Storage:
+    def __init__(self, context):
+        self.context = context
+        self.redis_client = self.get_redis_client()
+
+    async def get_detector_data(self, path):
+        data = await self.redis_client.get(f"thumbor-detector-{path}")
+        if data:
+            return loads(data)
+        return None
+
+    def get_redis_client(self):
+        redis_mode = str(self.context.config.REDIS_QUEUE_MODE).lower()
+
+        if redis_mode == SINGLE_NODE:
+            return self.__redis_single_node_client()
+        if redis_mode == SENTINEL:
+            return self.__redis_sentinel_client()
+
+        raise RedisError(
+            f"REDIS_QUEUE_MODE must be {SINGLE_NODE} or {SENTINEL}"
+        )
+
+    def __redis_single_node_client(self):
+        return Redis(
+            host=self.context.config.REDIS_QUEUE_SERVER_HOST,
+            port=self.context.config.REDIS_QUEUE_SERVER_PORT,
+            db=self.context.config.REDIS_QUEUE_SERVER_DB,
+            password=self.context.config.REDIS_QUEUE_SERVER_PASSWORD,
+        )
+
+    def __redis_sentinel_client(self):
+
+        instances_split = (
+            self.context.config.REDIS_QUEUE_SENTINEL_INSTANCES.split(",")
+        )
+        instances = [
+            tuple(instance.split(":", 1)) for instance in instances_split
+        ]
+
+        if self.context.config.REDIS_QUEUE_SENTINEL_PASSWORD:
+            sentinel_instance = Sentinel(
+                instances,
+                socket_timeout=self.context.config.REDIS_QUEUE_SENTINEL_SOCKET_TIMEOUT,
+                sentinel_kwargs={
+                    "password": self.context.config.REDIS_QUEUE_SENTINEL_PASSWORD
+                },
+            )
+        else:
+            sentinel_instance = Sentinel(
+                instances,
+                socket_timeout=self.context.config.REDIS_QUEUE_SENTINEL_SOCKET_TIMEOUT,
+            )
+
+        return sentinel_instance.master_for(
+            self.context.config.REDIS_QUEUE_SENTINEL_MASTER_INSTANCE,
+            socket_timeout=self.context.config.REDIS_QUEUE_SENTINEL_SOCKET_TIMEOUT,
+            password=self.context.config.REDIS_QUEUE_SENTINEL_MASTER_PASSWORD,
+            db=self.context.config.REDIS_QUEUE_SENTINEL_MASTER_DB,
+        )

setup.py

@@ -13,13 +13,15 @@
     "pytest-cov==3.*,>=3.0.0",
     "pytest-asyncio==0.*,>=0.18.0",
     "coverage==6.*,>=6.3.2",
+    "thumbor==7.*",
 ]
 
 RUNTIME_REQUIREMENTS = [
     "opencv-python-headless==4.*,>=4.2.0",
     "Pillow>=9.0.0",
     "pyres==1.*,>=1.5.0",
     "sentry-sdk==0.*,>=0.14.2",
+    "redis==4.*,>=4.2.0",
 ]
 
 setup(

tests/fixtures/redis-sentinel/sentinel-secure.conf

@@ -4,7 +4,7 @@ dir /tmp
 
 requirepass SENTINEL_PASSWORD
 sentinel resolve-hostnames yes
-sentinel announce_hostnames yes
+sentinel announce-hostnames yes
 sentinel monitor MASTER_INSTANCE localhost MASTER_PORT SENTINEL_QUORUM
 sentinel down-after-milliseconds MASTER_INSTANCE SENTINEL_DOWN_AFTER
 sentinel parallel-syncs MASTER_INSTANCE 1

tests/test_redis_storage.py

@@ -0,0 +1,73 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+
+import uuid
+
+from redis import RedisError
+
+from thumbor.testing import TestCase
+from tornado.testing import gen_test
+from preggy import expect
+
+import remotecv.storages.redis_storage
+
+
+class RedisStorageTestCase(TestCase):
+    @gen_test
+    async def test_should_be_none_when_not_available(self):
+        storage = remotecv.storages.redis_storage.Storage(self.context)
+        result = await storage.get_detector_data(uuid.uuid4())
+        expect(result).to_be_null()
+        self.assertIsNone(result)
+
+    @gen_test
+    async def test_should_be_points_when_available(self):
+        key = uuid.uuid4()
+        storage = remotecv.storages.redis_storage.Storage(self.context)
+        await storage.redis_client.set(f"thumbor-detector-{key}", '[{"x": 1}]')
+        result = await storage.get_detector_data(key)
+        expect(result).to_equal([{"x": 1}])
+
+    @gen_test
+    async def test_should_be_error_when_invalid_redis_mode(self):
+        self.context.config.REDIS_QUEUE_MODE = "invalid"
+        with self.assertRaises(RedisError):
+            remotecv.storages.redis_storage.Storage(self.context)
+
+    @gen_test
+    async def test_should_be_points_when_available_in_sentinal(self):
+        self.context.config.REDIS_QUEUE_MODE = "sentinel"
+        self.context.config.REDIS_QUEUE_SENTINEL_INSTANCES = "localhost:26379"
+        self.context.config.REDIS_QUEUE_SENTINEL_PASSWORD = None
+        self.context.config.REDIS_QUEUE_SENTINEL_MASTER_INSTANCE = (
+            "redismaster"
+        )
+        self.context.config.REDIS_QUEUE_SENTINEL_MASTER_PASSWORD = None
+        self.context.config.REDIS_QUEUE_SENTINEL_MASTER_DB = 0
+
+        key = uuid.uuid4()
+        storage = remotecv.storages.redis_storage.Storage(self.context)
+        await storage.redis_client.set(f"thumbor-detector-{key}", '[{"x": 1}]')
+        result = await storage.get_detector_data(key)
+        expect(result).to_equal([{"x": 1}])
+
+    @gen_test
+    async def test_should_be_points_when_available_in_sentinal_without_auth(
+        self,
+    ):
+        self.context.config.REDIS_QUEUE_MODE = "sentinel"
+        self.context.config.REDIS_QUEUE_SENTINEL_INSTANCES = "localhost:26380"
+        self.context.config.REDIS_QUEUE_SENTINEL_PASSWORD = "superpassword"
+        self.context.config.REDIS_QUEUE_SENTINEL_MASTER_INSTANCE = (
+            "redismaster"
+        )
+        self.context.config.REDIS_QUEUE_SENTINEL_MASTER_PASSWORD = None
+        self.context.config.REDIS_QUEUE_SENTINEL_MASTER_DB = 0
+
+        storage = remotecv.storages.redis_storage.Storage(self.context)
+        await storage.redis_client.set(
+            "thumbor-detector-random_path", '[{"x": 1}]'
+        )
+        result = await storage.get_detector_data("random_path")
+        expect(result).to_equal([{"x": 1}])