New Filtering System

.github/CODEOWNERS

@@ -1,21 +1,17 @@
 # Extensions
 **/bot/exts/backend/sync/**             @MarkKoz
-**/bot/exts/filters/*token_remover.py   @MarkKoz
 **/bot/exts/moderation/*silence.py      @MarkKoz
 bot/exts/info/codeblock/**              @MarkKoz
 bot/exts/utils/extensions.py            @MarkKoz
 bot/exts/utils/snekbox.py               @MarkKoz @jb3
 bot/exts/moderation/**                  @mbaruh @Den4200 @ks129 @jb3
 bot/exts/info/**                        @Den4200 @jb3
 bot/exts/info/information.py            @mbaruh @jb3
-bot/exts/filters/**                     @mbaruh @jb3
+bot/exts/filtering/**                   @mbaruh
 bot/exts/fun/**                         @ks129
 bot/exts/utils/**                       @ks129 @jb3
 bot/exts/recruitment/**                 @wookie184
 
-# Rules
-bot/rules/**                            @mbaruh
-
 # Utils
 bot/utils/function.py                   @MarkKoz
 bot/utils/lock.py                       @MarkKoz

bot/bot.py

@@ -1,5 +1,4 @@
 import asyncio
-from collections import defaultdict
 
 import aiohttp
 from pydis_core import BotBase
@@ -27,8 +26,6 @@ def __init__(self, *args, **kwargs):
 
         super().__init__(*args, **kwargs)
 
-        self.filter_list_cache = defaultdict(dict)
-
     async def ping_services(self) -> None:
         """A helper to make sure all the services the bot relies on are available on startup."""
         # Connect Site/API
@@ -45,33 +42,10 @@ async def ping_services(self) -> None:
                     raise
                 await asyncio.sleep(constants.URLs.connect_cooldown)
 
-    def insert_item_into_filter_list_cache(self, item: dict[str, str]) -> None:
-        """Add an item to the bots filter_list_cache."""
-        type_ = item["type"]
-        allowed = item["allowed"]
-        content = item["content"]
-
-        self.filter_list_cache[f"{type_}.{allowed}"][content] = {
-            "id": item["id"],
-            "comment": item["comment"],
-            "created_at": item["created_at"],
-            "updated_at": item["updated_at"],
-        }
-
-    async def cache_filter_list_data(self) -> None:
-        """Cache all the data in the FilterList on the site."""
-        full_cache = await self.api_client.get('bot/filter-lists')
-
-        for item in full_cache:
-            self.insert_item_into_filter_list_cache(item)
-
     async def setup_hook(self) -> None:
         """Default async initialisation method for discord.py."""
         await super().setup_hook()
 
-        # Build the FilterList cache
-        await self.cache_filter_list_data()
-
         # This is not awaited to avoid a deadlock with any cogs that have
         # wait_until_guild_available in their cog_load method.
         scheduling.create_task(self.load_extensions(exts))

bot/constants.py

@@ -336,43 +336,6 @@ class _Free(EnvConfig):
 Free = _Free()
 
 
-class Rule(BaseModel):
-    interval: int
-    max: int
-
-
-# Some help in choosing an appropriate name for this is appreciated
-class ExtendedRule(Rule):
-    max_consecutive: int
-
-
-class Rules(BaseModel):
-    attachments: Rule = Rule(interval=10, max=6)
-    burst: Rule = Rule(interval=10, max=7)
-    chars: Rule = Rule(interval=5, max=4_200)
-    discord_emojis: Rule = Rule(interval=10, max=20)
-    duplicates: Rule = Rule(interval=10, max=3)
-    links: Rule = Rule(interval=10, max=10)
-    mentions: Rule = Rule(interval=10, max=5)
-    newlines: ExtendedRule = ExtendedRule(interval=10, max=100, max_consecutive=10)
-    role_mentions: Rule = Rule(interval=10, max=3)
-
-
-class _AntiSpam(EnvConfig):
-    EnvConfig.Config.env_prefix = 'anti_spam_'
-
-    cache_size = 100
-
-    clean_offending = True
-    ping_everyone = True
-
-    remove_timeout_after = 600
-    rules = Rules()
-
-
-AntiSpam = _AntiSpam()
-
-
 class _HelpChannels(EnvConfig):
     EnvConfig.Config.env_prefix = "help_channels_"
 
@@ -659,47 +622,6 @@ class _Icons(EnvConfig):
 Icons = _Icons()
 
 
-class _Filter(EnvConfig):
-    EnvConfig.Config.env_prefix = "filters_"
-
-    filter_domains = True
-    filter_everyone_ping = True
-    filter_invites = True
-    filter_zalgo = False
-    watch_regex = True
-    watch_rich_embeds = True
-
-    # Notifications are not expected for "watchlist" type filters
-
-    notify_user_domains = False
-    notify_user_everyone_ping = True
-    notify_user_invites = True
-    notify_user_zalgo = False
-
-    offensive_msg_delete_days = 7
-    ping_everyone = True
-
-    channel_whitelist = [
-        Channels.admins,
-        Channels.big_brother,
-        Channels.dev_log,
-        Channels.message_log,
-        Channels.mod_log,
-        Channels.staff_lounge
-    ]
-    role_whitelist = [
-        Roles.admins,
-        Roles.helpers,
-        Roles.moderators,
-        Roles.owners,
-        Roles.python_community,
-        Roles.partners
-    ]
-
-
-Filter = _Filter()
-
-
 class _Keys(EnvConfig):
 
     EnvConfig.Config.env_prefix = "api_keys_"

bot/converters.py

@@ -9,7 +9,7 @@
 import discord
 from aiohttp import ClientConnectorError
 from dateutil.relativedelta import relativedelta
-from discord.ext.commands import BadArgument, Bot, Context, Converter, IDConverter, MemberConverter, UserConverter
+from discord.ext.commands import BadArgument, Context, Converter, IDConverter, MemberConverter, UserConverter
 from discord.utils import escape_markdown, snowflake_time
 from pydis_core.site_api import ResponseCodeError
 from pydis_core.utils import unqualify
@@ -68,54 +68,6 @@ async def convert(self, ctx: Context, server_invite: str) -> dict:
         raise BadArgument("This does not appear to be a valid Discord server invite.")
 
 
-class ValidFilterListType(Converter):
-    """
-    A converter that checks whether the given string is a valid FilterList type.
-
-    Raises `BadArgument` if the argument is not a valid FilterList type, and simply
-    passes through the given argument otherwise.
-    """
-
-    @staticmethod
-    async def get_valid_types(bot: Bot) -> list:
-        """
-        Try to get a list of valid filter list types.
-
-        Raise a BadArgument if the API can't respond.
-        """
-        try:
-            valid_types = await bot.api_client.get('bot/filter-lists/get-types')
-        except ResponseCodeError:
-            raise BadArgument("Cannot validate list_type: Unable to fetch valid types from API.")
-
-        return [enum for enum, classname in valid_types]
-
-    async def convert(self, ctx: Context, list_type: str) -> str:
-        """Checks whether the given string is a valid FilterList type."""
-        valid_types = await self.get_valid_types(ctx.bot)
-        list_type = list_type.upper()
-
-        if list_type not in valid_types:
-
-            # Maybe the user is using the plural form of this type,
-            # e.g. "guild_invites" instead of "guild_invite".
-            #
-            # This code will support the simple plural form (a single 's' at the end),
-            # which works for all current list types, but if a list type is added in the future
-            # which has an irregular plural form (like 'ies'), this code will need to be
-            # refactored to support this.
-            if list_type.endswith("S") and list_type[:-1] in valid_types:
-                list_type = list_type[:-1]
-
-            else:
-                valid_types_list = '\n'.join([f"• {type_.lower()}" for type_ in valid_types])
-                raise BadArgument(
-                    f"You have provided an invalid list type!\n\n"
-                    f"Please provide one of the following: \n{valid_types_list}"
-                )
-        return list_type
-
-
 class Extension(Converter):
     """
     Fully qualify the name of an extension and ensure it exists.

bot/exts/filtering/FILTERS-DEVELOPMENT.md

@@ -0,0 +1,63 @@
+# Filters Development
+This file gives a short overview of the extension, and shows how to perform some basic changes/additions to it.
+
+## Overview
+The main idea is that there is a list of filters each deciding whether they apply to the given content.
+For example, there can be a filter that decides it will trigger when the content contains the string "lemon".
+
+There are several types of filters, and two filters of the same type differ by their content.
+For example, filters of type "token" search for a specific token inside the provided string.
+One token filter might look for the string "lemon", while another will look for the string "joe".
+
+Each filter has a set of settings that decide when it triggers (e.g. in which channels, in which categories, etc.), and what happens if it does (e.g. delete the message, ping specific roles/users, etc.).
+Filters of a specific type can have additional settings that are special to them.
+
+A list of filters is contained within a filter list.
+The filter list gets content to filter, and dispatches it to each of its filters.
+It takes the answers from its filters and returns a unified response (e.g. if at least one of the filters says it should be deleted, then the filter list response will include it).
+
+A filter list has the same set of possible settings, which act as defaults.
+If a filter in the list doesn't define a value for a setting (meaning it has a value of None), it will use the value of the containing filter list.
+
+The cog receives "filtering events". For example, a new message is sent.
+It creates a "filtering context" with everything a filtering list needs to know to provide an answer for what should be done.
+For example, if the event is a new message, then the content to filter is the content of the message, embeds if any exist, etc.
+
+The cog dispatches the event to each filter list, gets the result from each, compiles them, and takes any action dictated by them.
+For example, if any of the filter lists want the message to be deleted, then the cog will delete it.
+
+## Example Changes
+### Creating a new type of filter list
+1. Head over to `bot.exts.filtering._filter_lists` and create a new Python file.
+2. Subclass the FilterList class in `bot.exts.filtering._filter_lists.filter_list` and implement its abstract methods. Make sure to set the `name` class attribute.
+
+You can now add filter lists to the database with the same name defined in the new FilterList subclass.
+
+### Creating a new type of filter
+1. Head over to `bot.exts.filtering._filters` and create a new Python file.
+2. Subclass the Filter class in `bot.exts.filtering._filters.filter` and implement its abstract methods.
+3. Make sure to set the `name` class attribute, and have one of the FilterList subclasses return this new Filter subclass in `get_filter_type`.
+
+### Creating a new type of setting
+1. Head over to `bot.exts.filtering._settings_types`, and open a new Python file in either `actions` or `validations`, depending on whether you want to subclass `ActionEntry` or `ValidationEntry`.
+2. Subclass one of the aforementioned classes, and implement its abstract methods. Make sure to set the `name` and `description` class attributes.
+
+You can now make the appropriate changes to the site repo:
+1. Add a new field in the `Filter` and `FilterList` models. Make sure that on `Filter` it's nullable, and on `FilterList` it isn't.
+2. In `serializers.py`, add the new field to `SETTINGS_FIELDS`, and to `ALLOW_BLANK_SETTINGS` or `ALLOW_EMPTY_SETTINGS` if appropriate. If it's not a part of any group of settings, add it `BASE_SETTINGS_FIELDS`, otherwise add it to the appropriate group or create a new one.
+3. If you created a new group, make sure it's used in `to_representation`.
+4. Update the docs in the filter viewsets.
+
+You can merge the changes to the bot first - if no such field is loaded from the database it'll just be ignored.
+
+You can define entries that are a group of fields in the database.
+In that case the created subclass should have fields whose names are the names of the fields in the database.
+Then, the description will be a dictionary, whose keys are the names of the fields, and values are the descriptions for each field.
+
+### Creating a new type of filtering event
+1. Head over to `bot.exts.filtering._filter_context` and add a new value to the `Event` enum.
+2. Implement the dispatching and actioning of the new event in the cog, by either adding it to an existing even listener, or creating a new one.
+3. Have the appropriate filter lists subscribe to the event, so they receive it.
+4. Have the appropriate unique filters (currently under `unique` and `antispam` in `bot.exts.filtering._filters`) subscribe to the event, so they receive it.
+
+It should be noted that the filtering events don't need to correspond to Discord events. For example, `nickname` isn't a Discord event and is dispatched when a message is sent.

bot/exts/filtering/_filter_context.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+import typing
+from collections.abc import Callable, Coroutine, Iterable
+from dataclasses import dataclass, field, replace
+from enum import Enum, auto
+
+import discord
+from discord import DMChannel, Embed, Member, Message, StageChannel, TextChannel, Thread, User, VoiceChannel
+
+from bot.utils.message_cache import MessageCache
+
+if typing.TYPE_CHECKING:
+    from bot.exts.filtering._filters.filter import Filter
+    from bot.exts.utils.snekbox._io import FileAttachment
+
+
+class Event(Enum):
+    """Types of events that can trigger filtering. Note this does not have to align with gateway event types."""
+
+    MESSAGE = auto()
+    MESSAGE_EDIT = auto()
+    NICKNAME = auto()
+    SNEKBOX = auto()
+
+
+@dataclass
+class FilterContext:
+    """A dataclass containing the information that should be filtered, and output information of the filtering."""
+
+    # Input context
+    event: Event  # The type of event
+    author: User | Member | None  # Who triggered the event
+    channel: TextChannel | VoiceChannel | StageChannel | Thread | DMChannel | None  # The channel involved
+    content: str | Iterable  # What actually needs filtering. The Iterable type depends on the filter list.
+    message: Message | None  # The message involved
+    embeds: list[Embed] = field(default_factory=list)  # Any embeds involved
+    attachments: list[discord.Attachment | FileAttachment] = field(default_factory=list)  # Any attachments sent.
+    before_message: Message | None = None
+    message_cache: MessageCache | None = None
+    # Output context
+    dm_content: str = ""  # The content to DM the invoker
+    dm_embed: str = ""  # The embed description to DM the invoker
+    send_alert: bool = False  # Whether to send an alert for the moderators
+    alert_content: str = ""  # The content of the alert
+    alert_embeds: list[Embed] = field(default_factory=list)  # Any embeds to add to the alert
+    action_descriptions: list[str] = field(default_factory=list)  # What actions were taken
+    matches: list[str] = field(default_factory=list)  # What exactly was found
+    notification_domain: str = ""  # A domain to send the user for context
+    filter_info: dict['Filter', str] = field(default_factory=dict)  # Additional info from a filter.
+    messages_deletion: bool = False  # Whether the messages were deleted. Can't upload deletion log otherwise.
+    blocked_exts: set[str] = field(default_factory=set)  # Any extensions blocked (used for snekbox)
+    # Additional actions to perform
+    additional_actions: list[Callable[[FilterContext], Coroutine]] = field(default_factory=list)
+    related_messages: set[Message] = field(default_factory=set)  # Deletion will include these.
+    related_channels: set[TextChannel | Thread | DMChannel] = field(default_factory=set)
+    uploaded_attachments: dict[int, list[str]] = field(default_factory=dict)  # Message ID to attachment URLs.
+    upload_deletion_logs: bool = True  # Whether it's allowed to upload deletion logs.
+
+    def __post_init__(self):
+        # If it's in the context of a DM channel, self.channel won't be None, but self.channel.guild will.
+        self.in_guild = self.channel is None or self.channel.guild is not None
+
+    @classmethod
+    def from_message(
+        cls, event: Event, message: Message, before: Message | None = None, cache: MessageCache | None = None
+    ) -> FilterContext:
+        """Create a filtering context from the attributes of a message."""
+        return cls(
+            event,
+            message.author,
+            message.channel,
+            message.content,
+            message,
+            message.embeds,
+            message.attachments,
+            before,
+            cache
+        )
+
+    def replace(self, **changes) -> FilterContext:
+        """Return a new context object assigning new values to the specified fields."""
+        return replace(self, **changes)

bot/exts/filtering/_filter_lists/__init__.py

@@ -0,0 +1,9 @@
+from os.path import dirname
+
+from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType, list_type_converter
+from bot.exts.filtering._utils import subclasses_in_package
+
+filter_list_types = subclasses_in_package(dirname(__file__), f"{__name__}.", FilterList)
+filter_list_types = {filter_list.name: filter_list for filter_list in filter_list_types}
+
+__all__ = [filter_list_types, FilterList, ListType, list_type_converter]