RedisCache - Data Persistence

Pipfile

@@ -4,25 +4,27 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-discord-py = "~=1.3.2"
+aio-pika = "~=6.1"
 aiodns = "~=2.0"
 aiohttp = "~=3.5"
-sphinx = "~=2.2"
-markdownify = "~=0.4"
-lxml = "~=4.4"
-pyyaml = "~=5.1"
+aioredis = "~=1.3.1"
+beautifulsoup4 = "~=4.9"
+colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"}
+coloredlogs = "~=14.0"
+deepdiff = "~=4.0"
+discord.py = "~=1.3.2"
+fakeredis = "~=1.4"
+feedparser = "~=5.2"
 fuzzywuzzy = "~=0.17"
-aio-pika = "~=6.1"
+lxml = "~=4.4"
+markdownify = "~=0.4"
+more_itertools = "~=8.2"
 python-dateutil = "~=2.8"
-deepdiff = "~=4.0"
+pyyaml = "~=5.1"
 requests = "~=2.22"
-more_itertools = "~=8.2"
 sentry-sdk = "~=0.14"
-coloredlogs = "~=14.0"
-colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"}
+sphinx = "~=2.2"
 statsd = "~=3.3"
-feedparser = "~=5.2"
-beautifulsoup4 = "~=4.9"
 
 [dev-packages]
 coverage = "~=5.0"

azure-pipelines.yml

@@ -22,6 +22,7 @@ jobs:
       REDDIT_CLIENT_ID: spam
       REDDIT_SECRET: ham
       WOLFRAM_API_KEY: baz
+      REDIS_PASSWORD: ''
 
     steps:
       - task: UsePythonVersion@0

bot/bot.py

@@ -5,7 +5,9 @@
 from typing import Optional
 
 import aiohttp
+import aioredis
 import discord
+import fakeredis.aioredis
 from discord.ext import commands
 from sentry_sdk import push_scope
 
@@ -28,6 +30,9 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         self.http_session: Optional[aiohttp.ClientSession] = None
+        self.redis_session: Optional[aioredis.Redis] = None
+        self.redis_ready = asyncio.Event()
+        self.redis_closed = False
         self.api_client = api.APIClient(loop=self.loop)
 
         self._connector = None
@@ -44,6 +49,30 @@ def __init__(self, *args, **kwargs):
 
         self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot")
 
+    async def _create_redis_session(self) -> None:
+        """
+        Create the Redis connection pool, and then open the redis event gate.
+
+        If constants.Redis.use_fakeredis is True, we'll set up a fake redis pool instead
+        of attempting to communicate with a real Redis server. This is useful because it
+        means contributors don't necessarily need to get Redis running locally just
+        to run the bot.
+
+        The fakeredis cache won't have persistence across restarts, but that
+        usually won't matter for local bot testing.
+        """
+        if constants.Redis.use_fakeredis:
+            log.info("Using fakeredis instead of communicating with a real Redis server.")
+            self.redis_session = await fakeredis.aioredis.create_redis_pool()
+        else:
+            self.redis_session = await aioredis.create_redis_pool(
+                address=(constants.Redis.host, constants.Redis.port),
+                password=constants.Redis.password,
+            )
+
+        self.redis_closed = False
+        self.redis_ready.set()
+
     def add_cog(self, cog: commands.Cog) -> None:
         """Adds a "cog" to the bot and logs the operation."""
         super().add_cog(cog)
@@ -78,14 +107,20 @@ async def close(self) -> None:
         if self.stats._transport:
             self.stats._transport.close()
 
+        if self.redis_session:
+            self.redis_closed = True
+            self.redis_session.close()
+            self.redis_ready.clear()
+            await self.redis_session.wait_closed()
+
     async def login(self, *args, **kwargs) -> None:
         """Re-create the connector and set up sessions before logging into Discord."""
         self._recreate()
         await self.stats.create_socket()
         await super().login(*args, **kwargs)
 
     def _recreate(self) -> None:
-        """Re-create the connector, aiohttp session, and the APIClient."""
+        """Re-create the connector, aiohttp session, the APIClient and the Redis session."""
         # Use asyncio for DNS resolution instead of threads so threads aren't spammed.
         # Doesn't seem to have any state with regards to being closed, so no need to worry?
         self._resolver = aiohttp.AsyncResolver()
@@ -96,6 +131,14 @@ def _recreate(self) -> None:
                 "The previous connector was not closed; it will remain open and be overwritten"
             )
 
+        if self.redis_session and not self.redis_session.closed:
+            log.warning(
+                "The previous redis pool was not closed; it will remain open and be overwritten"
+            )
+
+        # Create the redis session
+        self.loop.create_task(self._create_redis_session())
+
         # Use AF_INET as its socket family to prevent HTTPS related problems both locally
         # and in production.
         self._connector = aiohttp.TCPConnector(

bot/cogs/antispam.py

@@ -94,7 +94,7 @@ async def upload_messages(self, actor_id: int, modlog: ModLog) -> None:
         await modlog.send_log_message(
             icon_url=Icons.filtering,
             colour=Colour(Colours.soft_red),
-            title=f"Spam detected!",
+            title="Spam detected!",
             text=mod_alert_message,
             thumbnail=last_message.author.avatar_url_as(static_format="png"),
             channel_id=Channels.mod_alerts,
@@ -130,7 +130,7 @@ async def alert_on_validation_error(self) -> None:
             body += "\n\n**The cog has been unloaded.**"
 
             await self.mod_log.send_log_message(
-                title=f"Error: AntiSpam configuration validation failed!",
+                title="Error: AntiSpam configuration validation failed!",
                 text=body,
                 ping_everyone=True,
                 icon_url=Icons.token_removed,

bot/cogs/defcon.py

@@ -81,7 +81,7 @@ async def sync_settings(self) -> None:
             else:
                 self.enabled = False
                 self.days = timedelta(days=0)
-                log.info(f"DEFCON disabled")
+                log.info("DEFCON disabled")
 
             await self.update_channel_topic()
 

bot/cogs/duck_pond.py

@@ -117,7 +117,7 @@ async def relay_message(self, message: Message) -> None:
                     avatar_url=message.author.avatar_url
                 )
             except discord.HTTPException:
-                log.exception(f"Failed to send an attachment to the webhook")
+                log.exception("Failed to send an attachment to the webhook")
 
         await message.add_reaction("✅")
 

bot/cogs/help_channels.py

@@ -391,7 +391,7 @@ async def init_categories(self) -> None:
             self.in_use_category = await self.try_get_channel(constants.Categories.help_in_use)
             self.dormant_category = await self.try_get_channel(constants.Categories.help_dormant)
         except discord.HTTPException:
-            log.exception(f"Failed to get a category; cog will be removed")
+            log.exception("Failed to get a category; cog will be removed")
             self.bot.remove_cog(self.qualified_name)
 
     async def init_cog(self) -> None:

bot/cogs/moderation/scheduler.py

@@ -91,7 +91,7 @@ async def apply_infraction(
         log.trace(f"Applying {infr_type} infraction #{id_} to {user}.")
 
         # Default values for the confirmation message and mod log.
-        confirm_msg = f":ok_hand: applied"
+        confirm_msg = ":ok_hand: applied"
 
         # Specifying an expiry for a note or warning makes no sense.
         if infr_type in ("note", "warning"):
@@ -154,7 +154,7 @@ async def apply_infraction(
                     self.schedule_task(infraction["id"], infraction)
             except discord.HTTPException as e:
                 # Accordingly display that applying the infraction failed.
-                confirm_msg = f":x: failed to apply"
+                confirm_msg = ":x: failed to apply"
                 expiry_msg = ""
                 log_content = ctx.author.mention
                 log_title = "failed to apply"
@@ -281,7 +281,7 @@ async def pardon_infraction(
 
             log.warning(f"Failed to pardon {infr_type} infraction #{id_} for {user}.")
         else:
-            confirm_msg = f":ok_hand: pardoned"
+            confirm_msg = ":ok_hand: pardoned"
             log_title = "pardoned"
 
             log.info(f"Pardoned {infr_type} infraction #{id_} for {user}.")
@@ -353,7 +353,7 @@ async def deactivate_infraction(
                 )
         except discord.Forbidden:
             log.warning(f"Failed to deactivate infraction #{id_} ({type_}): bot lacks permissions.")
-            log_text["Failure"] = f"The bot lacks permissions to do this (role hierarchy?)"
+            log_text["Failure"] = "The bot lacks permissions to do this (role hierarchy?)"
             log_content = mod_role.mention
         except discord.HTTPException as e:
             log.exception(f"Failed to deactivate infraction #{id_} ({type_})")
@@ -402,7 +402,7 @@ async def deactivate_infraction(
 
         # Send a log message to the mod log.
         if send_log:
-            log_title = f"expiration failed" if "Failure" in log_text else "expired"
+            log_title = "expiration failed" if "Failure" in log_text else "expired"
 
             user = self.bot.get_user(user_id)
             avatar = user.avatar_url_as(static_format="png") if user else None

bot/cogs/moderation/silence.py

@@ -91,7 +91,7 @@ async def silence(self, ctx: Context, duration: HushDurationConverter = 10) -> N
 
         await ctx.send(f"{Emojis.check_mark} silenced current channel for {duration} minute(s).")
         await asyncio.sleep(duration*60)
-        log.info(f"Unsilencing channel after set delay.")
+        log.info("Unsilencing channel after set delay.")
         await ctx.invoke(self.unsilence)
 
     @commands.command(aliases=("unhush",))

bot/cogs/stats.py

@@ -68,15 +68,15 @@ async def on_member_join(self, member: Member) -> None:
         if member.guild.id != Guild.id:
             return
 
-        self.bot.stats.gauge(f"guild.total_members", len(member.guild.members))
+        self.bot.stats.gauge("guild.total_members", len(member.guild.members))
 
     @Cog.listener()
     async def on_member_leave(self, member: Member) -> None:
         """Update member count stat on member leave."""
         if member.guild.id != Guild.id:
             return
 
-        self.bot.stats.gauge(f"guild.total_members", len(member.guild.members))
+        self.bot.stats.gauge("guild.total_members", len(member.guild.members))
 
     @Cog.listener()
     async def on_member_update(self, _before: Member, after: Member) -> None:

bot/cogs/utils.py

@@ -253,8 +253,8 @@ async def vote(self, ctx: Context, title: str, *options: str) -> None:
     async def send_pep_zero(self, ctx: Context) -> None:
         """Send information about PEP 0."""
         pep_embed = Embed(
-            title=f"**PEP 0 - Index of Python Enhancement Proposals (PEPs)**",
-            description=f"[Link](https://www.python.org/dev/peps/)"
+            title="**PEP 0 - Index of Python Enhancement Proposals (PEPs)**",
+            description="[Link](https://www.python.org/dev/peps/)"
         )
         pep_embed.set_thumbnail(url=ICON_URL)
         pep_embed.add_field(name="Status", value="Active")

bot/cogs/watchchannels/talentpool.py

@@ -61,7 +61,7 @@ async def watch_command(self, ctx: Context, user: FetchedMember, *, reason: str)
             return
 
         if isinstance(user, Member) and any(role.id in STAFF_ROLES for role in user.roles):
-            await ctx.send(f":x: Nominating staff members, eh? Here's a cookie :cookie:")
+            await ctx.send(":x: Nominating staff members, eh? Here's a cookie :cookie:")
             return
 
         if not await self.fetch_user_cache():

bot/cogs/watchchannels/watchchannel.py

@@ -82,7 +82,7 @@ def consuming_messages(self) -> bool:
             exc = self._consume_task.exception()
             if exc:
                 self.log.exception(
-                    f"The message queue consume task has failed with:",
+                    "The message queue consume task has failed with:",
                     exc_info=exc
                 )
             return False
@@ -146,7 +146,7 @@ async def fetch_user_cache(self) -> bool:
         try:
             data = await self.bot.api_client.get(self.api_endpoint, params=self.api_default_params)
         except ResponseCodeError as err:
-            self.log.exception(f"Failed to fetch the watched users from the API", exc_info=err)
+            self.log.exception("Failed to fetch the watched users from the API", exc_info=err)
             return False
 
         self.watched_users = defaultdict(dict)
@@ -173,7 +173,7 @@ async def consume_messages(self, delay_consumption: bool = True) -> None:
             self.log.trace(f"Sleeping {BigBrotherConfig.log_delay} seconds before consuming message queue")
             await asyncio.sleep(BigBrotherConfig.log_delay)
 
-        self.log.trace(f"Started consuming the message queue")
+        self.log.trace("Started consuming the message queue")
 
         # If the previous consumption Task failed, first consume the existing comsumption_queue
         if not self.consumption_queue:
@@ -208,7 +208,7 @@ async def webhook_send(
             await self.webhook.send(content=content, username=username, avatar_url=avatar_url, embed=embed)
         except discord.HTTPException as exc:
             self.log.exception(
-                f"Failed to send a message to the webhook",
+                "Failed to send a message to the webhook",
                 exc_info=exc
             )
 
@@ -254,7 +254,7 @@ async def relay_message(self, msg: Message) -> None:
                 )
             except discord.HTTPException as exc:
                 self.log.exception(
-                    f"Failed to send an attachment to the webhook",
+                    "Failed to send an attachment to the webhook",
                     exc_info=exc
                 )
 
@@ -326,13 +326,13 @@ def _remove_user(self, user_id: int) -> None:
 
     def cog_unload(self) -> None:
         """Takes care of unloading the cog and canceling the consumption task."""
-        self.log.trace(f"Unloading the cog")
+        self.log.trace("Unloading the cog")
         if self._consume_task and not self._consume_task.done():
             self._consume_task.cancel()
             try:
                 self._consume_task.result()
             except asyncio.CancelledError as e:
                 self.log.exception(
-                    f"The consume task was canceled. Messages may be lost.",
+                    "The consume task was canceled. Messages may be lost.",
                     exc_info=e
                 )

bot/constants.py

@@ -15,7 +15,7 @@
 from collections.abc import Mapping
 from enum import Enum
 from pathlib import Path
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 import yaml
 
@@ -198,7 +198,18 @@ class Bot(metaclass=YAMLGetter):
 
     prefix: str
     token: str
-    sentry_dsn: str
+    sentry_dsn: Optional[str]
+
+
+class Redis(metaclass=YAMLGetter):
+    section = "bot"
+    subsection = "redis"
+
+    host: str
+    port: int
+    password: Optional[str]
+    use_fakeredis: bool  # If this is True, Bot will use fakeredis.aioredis
+
 
 class Filter(metaclass=YAMLGetter):
     section = "filter"
@@ -450,7 +461,7 @@ class Guild(metaclass=YAMLGetter):
 class Keys(metaclass=YAMLGetter):
     section = "keys"
 
-    site_api: str
+    site_api: Optional[str]
 
 
 class URLs(metaclass=YAMLGetter):
@@ -493,16 +504,16 @@ class Reddit(metaclass=YAMLGetter):
     section = "reddit"
 
     subreddits: list
-    client_id: str
-    secret: str
+    client_id: Optional[str]
+    secret: Optional[str]
 
 
 class Wolfram(metaclass=YAMLGetter):
     section = "wolfram"
 
     user_limit_day: int
     guild_limit_day: int
-    key: str
+    key: Optional[str]
 
 
 class AntiSpam(metaclass=YAMLGetter):