Update to Python 3.10 and async-rediscache v1.0.0

.github/workflows/lint-test.yml

@@ -61,7 +61,7 @@ jobs:
         id: python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.10'
 
       # This step caches our Python dependencies. To make sure we
       # only restore a cache when the dependencies, the python version,

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 python:3.9-slim
+FROM --platform=linux/amd64 python:3.10-slim
 
 # Set pip to have no saved cache
 ENV PIP_NO_CACHE_DIR=false \

bot/__main__.py

@@ -6,6 +6,7 @@
 from botcore import StartupError
 from botcore.site_api import APIClient
 from discord.ext import commands
+from redis import RedisError
 
 import bot
 from bot import constants
@@ -19,18 +20,17 @@
 async def _create_redis_session() -> RedisSession:
     """Create and connect to a redis session."""
     redis_session = RedisSession(
-        address=(constants.Redis.host, constants.Redis.port),
+        host=constants.Redis.host,
+        port=constants.Redis.port,
         password=constants.Redis.password,
-        minsize=1,
-        maxsize=20,
+        max_connections=20,
         use_fakeredis=constants.Redis.use_fakeredis,
         global_namespace="bot",
     )
     try:
-        await redis_session.connect()
-    except OSError as e:
+        return await redis_session.connect()
+    except RedisError as e:
         raise StartupError(e)
-    return redis_session
 
 
 async def main() -> None:

bot/exts/backend/sync/_syncers.py

@@ -154,8 +154,8 @@ async def _get_diff(guild: Guild) -> _Diff:
 
             def maybe_update(db_field: str, guild_value: t.Union[str, int]) -> None:
                 # Equalize DB user and guild user attributes.
-                if db_user[db_field] != guild_value:
-                    updated_fields[db_field] = guild_value
+                if db_user[db_field] != guild_value:  # noqa: B023
+                    updated_fields[db_field] = guild_value  # noqa: B023
 
             guild_user = guild.get_member(db_user["id"])
             if not guild_user and db_user["in_guild"]:

bot/exts/filters/antispam.py

@@ -185,7 +185,7 @@ async def on_message(self, message: Message) -> None:
             # Create a list of messages that were sent in the interval that the rule cares about.
             latest_interesting_stamp = arrow.utcnow() - timedelta(seconds=rule_config['interval'])
             messages_for_rule = list(
-                takewhile(lambda msg: msg.created_at > latest_interesting_stamp, relevant_messages)
+                takewhile(lambda msg: msg.created_at > latest_interesting_stamp, relevant_messages)  # noqa: B023
             )
 
             result = await rule_function(message, messages_for_rule, rule_config)

bot/exts/info/doc/_redis_cache.py

@@ -5,26 +5,33 @@
 import time
 from typing import Optional, TYPE_CHECKING
 
-from async_rediscache.types.base import RedisObject, namespace_lock
+from async_rediscache.types.base import RedisObject
 
 from bot.log import get_logger
+from bot.utils.lock import lock
 
 if TYPE_CHECKING:
     from ._cog import DocItem
 
-WEEK_SECONDS = datetime.timedelta(weeks=1).total_seconds()
+WEEK_SECONDS = int(datetime.timedelta(weeks=1).total_seconds())
 
 log = get_logger(__name__)
 
 
+def serialize_resource_id_from_doc_item(bound_args: dict) -> str:
+    """Return the redis_key of the DocItem `item` from the bound args of DocRedisCache.set."""
+    item: DocItem = bound_args["item"]
+    return f"doc:{item_key(item)}"
+
+
 class DocRedisCache(RedisObject):
     """Interface for redis functionality needed by the Doc cog."""
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._set_expires = dict[str, float]()
 
-    @namespace_lock
+    @lock("DocRedisCache.set", serialize_resource_id_from_doc_item, wait=True)
     async def set(self, item: DocItem, value: str) -> None:
         """
         Set the Markdown `value` for the symbol `item`.
@@ -34,83 +41,75 @@ async def set(self, item: DocItem, value: str) -> None:
         redis_key = f"{self.namespace}:{item_key(item)}"
         needs_expire = False
 
-        with await self._get_pool_connection() as connection:
-            set_expire = self._set_expires.get(redis_key)
-            if set_expire is None:
-                # An expire is only set if the key didn't exist before.
-                ttl = await connection.ttl(redis_key)
-                log.debug(f"Checked TTL for `{redis_key}`.")
-
-                if ttl == -1:
-                    log.warning(f"Key `{redis_key}` had no expire set.")
-                if ttl < 0:  # not set or didn't exist
-                    needs_expire = True
-                else:
-                    log.debug(f"Key `{redis_key}` has a {ttl} TTL.")
-                    self._set_expires[redis_key] = time.monotonic() + ttl - .1  # we need this to expire before redis
-
-            elif time.monotonic() > set_expire:
-                # If we got here the key expired in redis and we can be sure it doesn't exist.
+        set_expire = self._set_expires.get(redis_key)
+        if set_expire is None:
+            # An expire is only set if the key didn't exist before.
+            ttl = await self.redis_session.client.ttl(redis_key)
+            log.debug(f"Checked TTL for `{redis_key}`.")
+
+            if ttl == -1:
+                log.warning(f"Key `{redis_key}` had no expire set.")
+            if ttl < 0:  # not set or didn't exist
                 needs_expire = True
-                log.debug(f"Key `{redis_key}` expired in internal key cache.")
+            else:
+                log.debug(f"Key `{redis_key}` has a {ttl} TTL.")
+                self._set_expires[redis_key] = time.monotonic() + ttl - .1  # we need this to expire before redis
+
+        elif time.monotonic() > set_expire:
+            # If we got here the key expired in redis and we can be sure it doesn't exist.
+            needs_expire = True
+            log.debug(f"Key `{redis_key}` expired in internal key cache.")
 
-            await connection.hset(redis_key, item.symbol_id, value)
-            if needs_expire:
-                self._set_expires[redis_key] = time.monotonic() + WEEK_SECONDS
-                await connection.expire(redis_key, WEEK_SECONDS)
-                log.info(f"Set {redis_key} to expire in a week.")
+        await self.redis_session.client.hset(redis_key, item.symbol_id, value)
+        if needs_expire:
+            self._set_expires[redis_key] = time.monotonic() + WEEK_SECONDS
+            await self.redis_session.client.expire(redis_key, WEEK_SECONDS)
+            log.info(f"Set {redis_key} to expire in a week.")
 
-    @namespace_lock
     async def get(self, item: DocItem) -> Optional[str]:
         """Return the Markdown content of the symbol `item` if it exists."""
-        with await self._get_pool_connection() as connection:
-            return await connection.hget(f"{self.namespace}:{item_key(item)}", item.symbol_id, encoding="utf8")
+        return await self.redis_session.client.hget(f"{self.namespace}:{item_key(item)}", item.symbol_id)
 
-    @namespace_lock
     async def delete(self, package: str) -> bool:
         """Remove all values for `package`; return True if at least one key was deleted, False otherwise."""
         pattern = f"{self.namespace}:{package}:*"
 
-        with await self._get_pool_connection() as connection:
-            package_keys = [
-                package_key async for package_key in connection.iscan(match=pattern)
-            ]
-            if package_keys:
-                await connection.delete(*package_keys)
-                log.info(f"Deleted keys from redis: {package_keys}.")
-                self._set_expires = {
-                    key: expire for key, expire in self._set_expires.items() if not fnmatch.fnmatchcase(key, pattern)
-                }
-                return True
-            return False
+        package_keys = [
+            package_key async for package_key in self.redis_session.client.scan_iter(match=pattern)
+        ]
+        if package_keys:
+            await self.redis_session.client.delete(*package_keys)
+            log.info(f"Deleted keys from redis: {package_keys}.")
+            self._set_expires = {
+                key: expire for key, expire in self._set_expires.items() if not fnmatch.fnmatchcase(key, pattern)
+            }
+            return True
+        return False
 
 
 class StaleItemCounter(RedisObject):
     """Manage increment counters for stale `DocItem`s."""
 
-    @namespace_lock
     async def increment_for(self, item: DocItem) -> int:
         """
         Increment the counter for `item` by 1, set it to expire in 3 weeks and return the new value.
 
         If the counter didn't exist, initialize it with 1.
         """
         key = f"{self.namespace}:{item_key(item)}:{item.symbol_id}"
-        with await self._get_pool_connection() as connection:
-            await connection.expire(key, WEEK_SECONDS * 3)
-            return int(await connection.incr(key))
+        await self.redis_session.client.expire(key, WEEK_SECONDS * 3)
+        return int(await self.redis_session.client.incr(key))
 
-    @namespace_lock
     async def delete(self, package: str) -> bool:
         """Remove all values for `package`; return True if at least one key was deleted, False otherwise."""
-        with await self._get_pool_connection() as connection:
-            package_keys = [
-                package_key async for package_key in connection.iscan(match=f"{self.namespace}:{package}:*")
-            ]
-            if package_keys:
-                await connection.delete(*package_keys)
-                return True
-            return False
+        package_keys = [
+            package_key
+            async for package_key in self.redis_session.client.scan_iter(match=f"{self.namespace}:{package}:*")
+        ]
+        if package_keys:
+            await self.redis_session.client.delete(*package_keys)
+            return True
+        return False
 
 
 def item_key(item: DocItem) -> str:

bot/exts/moderation/defcon.py

@@ -6,14 +6,14 @@
 from typing import Optional, Union
 
 import arrow
-from aioredis import RedisError
 from async_rediscache import RedisCache
 from botcore.utils import scheduling
 from botcore.utils.scheduling import Scheduler
 from dateutil.relativedelta import relativedelta
 from discord import Colour, Embed, Forbidden, Member, TextChannel, User
 from discord.ext import tasks
 from discord.ext.commands import Cog, Context, group, has_any_role
+from redis import RedisError
 
 from bot.bot import Bot
 from bot.constants import Channels, Colours, Emojis, Event, Icons, MODERATION_ROLES, Roles

docker-compose.yml

@@ -108,7 +108,7 @@ services:
     depends_on:
       - web
       - redis
-      - snekbox
+      - snekbox-311
     env_file:
       - .env
     environment: